""" simpleTAL Interpreter Copyright (c) 2009 Colin Stewart (http://www.owlfish.com/) All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. 3. The name of the author may not be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. If you make any bug fixes or feature enhancements please let me know! The classes in this module implement the TAL language, expanding both XML and HTML templates. Module Dependencies: logging, simpleTALES, simpleTALTemplates """ import logging import xml.sax, xml.sax.saxutils,html, io, re, types, codecs import simpletal, copy, sys from . import FixedHTMLParser, sgmlentitynames __version__ = simpletal.__version__ try: # Is PyXML's LexicalHandler available? from xml.sax.saxlib import LexicalHandler use_lexical_handler = 1 except ImportError: use_lexical_handler = 0 class LexicalHandler: pass try: # Is PyXML's DOM2SAX available? import xml.dom.ext.Dom2Sax use_dom2sax = 1 except ImportError: use_dom2sax = 0 # Check Python version. If 3.3 or later, all SGML entities will be expanded in the HTML Parser if sys.version_info[0] > 3 or (sys.version_info[0] == 3 and sys.version_info[1] > 3): HTML_ENTITIES_PRE_EXPANDED = True else: HTML_ENTITIES_PRE_EXPANDED = False from . import simpleTALES # Name-space URIs METAL_NAME_URI="http://xml.zope.org/namespaces/metal" TAL_NAME_URI="http://xml.zope.org/namespaces/tal" # All commands are of the form (opcode, args, commandList) # The numbers are the opcodes, and also the order of priority # Argument: [(isLocalFlag (Y/n), variableName, variablePath),...] TAL_DEFINE = 1 # Argument: expression, endTagSymbol TAL_CONDITION = 2 # Argument: (varname, expression, endTagSymbol) TAL_REPEAT = 3 # Argument: (replaceFlag, type, expression) TAL_CONTENT = 4 # Not used in byte code, only ordering. TAL_REPLACE = 5 # Argument: [(attributeName, expression)] TAL_ATTRIBUTES = 6 # Argument: expression TAL_OMITTAG = 7 # Argument: (originalAttributeList, currentAttributeList) TAL_START_SCOPE = 8 # Argument: String to output TAL_OUTPUT = 9 # Argument: None TAL_STARTTAG = 10 # Argument: Tag, omitTagFlag TAL_ENDTAG_ENDSCOPE = 11 # Argument: None TAL_NOOP = 13 # METAL Starts here # Argument: expression, slotParams, endTagSymbol METAL_USE_MACRO = 14 # Argument: macroName, endTagSymbol METAL_DEFINE_SLOT=15 # Only used for parsing METAL_FILL_SLOT=16 METAL_DEFINE_MACRO=17 METAL_NAME_REGEX = re.compile ("[a-zA-Z_][a-zA-Z0-9_]*") SINGLETON_XML_REGEX = re.compile ('^<[^\s/>]+(?:\s*[^=>]+="[^">]+")*\s*/>') SINGLETON_BYTES_XML_REGEX = re.compile (b'^<[^\s/>]+(?:\s*[^=>]+="[^">]+")*\s*/>') ENTITY_REF_REGEX = re.compile (r'(?:&[a-zA-Z][\-\.a-zA-Z0-9]*[^\-\.a-zA-Z0-9])|(?:&#[xX]?[a-eA-E0-9]*[^0-9a-eA-E])') # The list of elements in HTML that can not have end tags - done as a dictionary for fast # lookup. HTML_FORBIDDEN_ENDTAG = {'AREA': 1, 'BASE': 1, 'BASEFONT': 1, 'BR': 1, 'COL': 1 ,'FRAME': 1, 'HR': 1, 'IMG': 1, 'INPUT': 1, 'ISINDEX': 1 ,'LINK': 1, 'META': 1, 'PARAM': 1} # List of element:attribute pairs that can use minimized form in HTML HTML_BOOLEAN_ATTS = {'AREA:NOHREF': 1, 'IMG:ISMAP': 1, 'OBJECT:DECLARE': 1 , 'INPUT:CHECKED': 1, 'INPUT:DISABLED': 1, 'INPUT:READONLY': 1, 'INPUT:ISMAP': 1 , 'SELECT:MULTIPLE': 1, 'SELECT:DISABLED': 1 , 'OPTGROUP:DISABLED': 1 , 'OPTION:SELECTED': 1, 'OPTION:DISABLED': 1 , 'TEXTAREA:DISABLED': 1, 'TEXTAREA:READONLY': 1 , 'BUTTON:DISABLED': 1, 'SCRIPT:DEFER': 1} class TemplateInterpreter: def __init__ (self): self.programStack = [] self.commandList = None self.symbolTable = None self.slotParameters = {} self.commandHandler = {} self.commandHandler [TAL_DEFINE] = self.cmdDefine self.commandHandler [TAL_CONDITION] = self.cmdCondition self.commandHandler [TAL_REPEAT] = self.cmdRepeat self.commandHandler [TAL_CONTENT] = self.cmdContent self.commandHandler [TAL_ATTRIBUTES] = self.cmdAttributes self.commandHandler [TAL_OMITTAG] = self.cmdOmitTag self.commandHandler [TAL_START_SCOPE] = self.cmdStartScope self.commandHandler [TAL_OUTPUT] = self.cmdOutput self.commandHandler [TAL_STARTTAG] = self.cmdOutputStartTag self.commandHandler [TAL_ENDTAG_ENDSCOPE] = self.cmdEndTagEndScope self.commandHandler [METAL_USE_MACRO] = self.cmdUseMacro self.commandHandler [METAL_DEFINE_SLOT] = self.cmdDefineSlot self.commandHandler [TAL_NOOP] = self.cmdNoOp def tagAsText (self, tagObj, singletonFlag=0): """ This returns a tag as text. """ tag,atts = tagObj result = ["<"] result.append (tag) for attName, attValue in atts: result.append (' ') result.append (attName) result.append ('="') result.append (html.escape (attValue, quote=1)) result.append ('"') if (singletonFlag): result.append (" />") else: result.append (">") return "".join (result) def initialise (self, context, outputFile): self.context = context self.file = outputFile def cleanState (self): self.scopeStack = [] self.programCounter = 0 self.movePCForward = None self.movePCBack = None self.outputTag = 1 self.originalAttributes = {} self.currentAttributes = [] # Used in repeat only. self.repeatAttributesCopy = [] self.currentSlots = {} self.repeatVariable = None self.tagContent = None # tagState flag as to whether there are any local variables to pop self.localVarsDefined = 0 # Pass in the parameters self.currentSlots = self.slotParameters def popProgram (self): vars, self.commandList, self.symbolTable = self.programStack.pop() self.programCounter,self.scopeStack,self.slotParameters,self.currentSlots, self.movePCForward,self.movePCBack,self.outputTag,self.originalAttributes,self.currentAttributes,self.repeatVariable,self.repeatAttributesCopy,self.tagContent,self.localVarsDefined = vars def pushProgram (self): vars = (self.programCounter ,self.scopeStack ,self.slotParameters ,self.currentSlots ,self.movePCForward ,self.movePCBack ,self.outputTag ,self.originalAttributes ,self.currentAttributes ,self.repeatVariable ,self.repeatAttributesCopy ,self.tagContent ,self.localVarsDefined) self.programStack.append ((vars,self.commandList, self.symbolTable)) def execute (self, template): self.cleanState() self.commandList, self.programCounter, programLength, self.symbolTable = template.getProgram() cmndList = self.commandList while (self.programCounter < programLength): cmnd = cmndList [self.programCounter] #print "PC: %s - Executing command: %s" % (str (self.programCounter), str (cmnd)) self.commandHandler[cmnd[0]] (cmnd[0], cmnd[1]) def cmdDefine (self, command, args): """ args: [(isLocalFlag (Y/n), variableName, variablePath),...] Define variables in either the local or global context """ foundLocals = 0 for isLocal, varName, varPath in args: result = self.context.evaluate (varPath, self.originalAttributes) if (isLocal): if (not foundLocals): foundLocals = 1 self.context.pushLocals () self.context.setLocal (varName, result) else: self.context.addGlobal (varName, result) self.localVarsDefined = foundLocals self.programCounter += 1 def cmdCondition (self, command, args): """ args: expression, endTagSymbol Conditionally continues with execution of all content contained by it. """ result = self.context.evaluate (args[0], self.originalAttributes) #~ if (result is None or (not result)): conditionFalse = 0 if (result is None): conditionFalse = 1 else: if (not result): conditionFalse = 1 try: temp = len (result) if (temp == 0): conditionFalse = 1 except: # Result is not a sequence. pass if (conditionFalse): # Nothing to output - evaluated to false. self.outputTag = 0 self.tagContent = None self.programCounter = self.symbolTable[args[1]] return self.programCounter += 1 def cmdRepeat (self, command, args): """ args: (varName, expression, endTagSymbol) Repeats anything in the cmndList """ if (self.repeatVariable is not None): # We are already part way through a repeat # Restore any attributes that might have been changed. if (self.currentAttributes != self.repeatAttributesCopy): self.currentAttributes = copy.copy (self.repeatAttributesCopy) self.outputTag = 1 self.tagContent = None self.movePCForward = None try: self.repeatVariable.increment() self.context.setLocal (args[0], self.repeatVariable.getCurrentValue()) self.programCounter += 1 return except IndexError as e: # We have finished the repeat self.repeatVariable = None self.context.removeRepeat (args[0]) # The locals were pushed in context.addRepeat self.context.popLocals() self.movePCBack = None # Suppress the final close tag and content self.tagContent = None self.outputTag = 0 self.programCounter = self.symbolTable [args[2]] # Restore the state of repeatAttributesCopy in case we are nested. self.repeatAttributesCopy = self.scopeStack.pop() return # The first time through this command result = self.context.evaluate (args[1], self.originalAttributes) if (result is not None and result == simpleTALES.DEFAULTVALUE): # Leave everything un-touched. self.programCounter += 1 return try: # We have three options, either the result is a natural sequence, an iterator, or something that can produce an iterator. isSequence = len (result) if (isSequence): # Only setup if we have a sequence with length self.repeatVariable = simpleTALES.RepeatVariable (result) else: # Delete the tags and their contents self.outputTag = 0 self.programCounter = self.symbolTable [args[2]] return except: # Not a natural sequence, can it produce an iterator? if (hasattr (result, "__iter__") and hasattr (result.__iter__, "__call__")): # We can get an iterator! self.repeatVariable = simpleTALES.IteratorRepeatVariable (result.__iter__()) elif (hasattr (result, "__next__") and hasattr (result.__next__, "__call__")): # Treat as an iterator self.repeatVariable = simpleTALES.IteratorRepeatVariable (result) else: # Just a plain object, let's not loop # Delete the tags and their contents self.outputTag = 0 self.programCounter = self.symbolTable [args[2]] return try: curValue = self.repeatVariable.getCurrentValue() except IndexError as e: # The iterator ran out of values before we started - treat as an empty list self.outputTag = 0 self.repeatVariable = None self.programCounter = self.symbolTable [args[2]] return # We really do want to repeat - so lets do it self.movePCBack = self.programCounter self.context.addRepeat (args[0], self.repeatVariable, curValue) # We keep the old state of the repeatAttributesCopy for nested loops self.scopeStack.append (self.repeatAttributesCopy) # Keep a copy of the current attributes for this tag self.repeatAttributesCopy = copy.copy (self.currentAttributes) self.programCounter += 1 def cmdContent (self, command, args): """ args: (replaceFlag, structureFlag, expression, endTagSymbol) Expands content """ result = self.context.evaluate (args[2], self.originalAttributes) if (result is None): if (args[0]): # Only output tags if this is a content not a replace self.outputTag = 0 # Output none of our content or the existing content, but potentially the tags self.movePCForward = self.symbolTable [args[3]] self.programCounter += 1 return elif (not result == simpleTALES.DEFAULTVALUE): # We have content, so let's suppress the natural content and output this! if (args[0]): self.outputTag = 0 self.tagContent = (args[1], result) self.movePCForward = self.symbolTable [args[3]] self.programCounter += 1 return else: # Default, let's just run through as normal self.programCounter += 1 return def cmdAttributes (self, command, args): """ args: [(attributeName, expression)] Add, leave, or remove attributes from the start tag """ attsToRemove = {} newAtts = [] for attName, attExpr in args: resultVal = self.context.evaluate (attExpr, self.originalAttributes) if (resultVal is None): # Remove this attribute from the current attributes attsToRemove [attName]=1 elif (not resultVal == simpleTALES.DEFAULTVALUE): # We have a value - let's use it! attsToRemove [attName]=1 if (isinstance (resultVal, str)): escapedAttVal = resultVal elif (isinstance (resultVal, bytes)): # THIS IS NOT A BUG! # Use Unicode in the Context object if you are not using Ascii escapedAttVal = str (resultVal, 'ascii') else: # THIS IS NOT A BUG! # Use Unicode in the Context object if you are not using Ascii escapedAttVal = str (resultVal) newAtts.append ((attName, escapedAttVal)) # Copy over the old attributes for oldAttName, oldAttValue in self.currentAttributes: if (not oldAttName in attsToRemove): newAtts.append ((oldAttName, oldAttValue)) self.currentAttributes = newAtts # Evaluate all other commands self.programCounter += 1 def cmdOmitTag (self, command, args): """ args: expression Conditionally turn off tag output """ result = self.context.evaluate (args, self.originalAttributes) if (result is not None and result): # Turn tag output off self.outputTag = 0 self.programCounter += 1 def cmdOutputStartTag (self, command, args): # Args: tagName tagName, singletonTag = args if (self.outputTag): if (self.tagContent is None and singletonTag): self.file.write (self.tagAsText ((tagName, self.currentAttributes), 1)) else: self.file.write (self.tagAsText ((tagName, self.currentAttributes))) if (self.movePCForward is not None): self.programCounter = self.movePCForward return self.programCounter += 1 return def cmdEndTagEndScope (self, command, args): # Args: tagName, omitFlag, singletonTag if (self.tagContent is not None): contentType, resultVal = self.tagContent if (contentType): if (isinstance (resultVal, Template)): # We have another template in the context, evaluate it! # Save our state! self.pushProgram() resultVal.expandInline (self.context, self.file, self) # Restore state self.popProgram() # End of the macro expansion (if any) so clear the parameters self.slotParameters = {} else: if (isinstance (resultVal, str)): self.file.write (resultVal) elif (isinstance (resultVal, bytes)): # THIS IS NOT A BUG! # Use Unicode in the Context object if you are not using Ascii self.file.write (str (resultVal, 'ascii')) else: # THIS IS NOT A BUG! # Use Unicode in the Context object if you are not using Ascii self.file.write (str (resultVal)) else: if (isinstance (resultVal, str)): self.file.write (html.escape (resultVal, quote=False)) elif (isinstance (resultVal, bytes)): # THIS IS NOT A BUG! # Use Unicode in the Context object if you are not using Ascii self.file.write (html.escape (str (resultVal, 'ascii'), quote=False)) else: # THIS IS NOT A BUG! # Use Unicode in the Context object if you are not using Ascii self.file.write (html.escape (str (resultVal), quote=False)) if (self.outputTag and not args[1]): # Do NOT output end tag if a singleton with no content if not (args[2] and self.tagContent is None): self.file.write ('') if (self.movePCBack is not None): self.programCounter = self.movePCBack return if (self.localVarsDefined): self.context.popLocals() self.movePCForward,self.movePCBack,self.outputTag,self.originalAttributes,self.currentAttributes,self.repeatVariable,self.tagContent,self.localVarsDefined = self.scopeStack.pop() self.programCounter += 1 def cmdOutput (self, command, args): self.file.write (args) self.programCounter += 1 def cmdStartScope (self, command, args): """ args: (originalAttributes, currentAttributes) Pushes the current state onto the stack, and sets up the new state """ self.scopeStack.append ((self.movePCForward ,self.movePCBack ,self.outputTag ,self.originalAttributes ,self.currentAttributes ,self.repeatVariable ,self.tagContent ,self.localVarsDefined)) self.movePCForward = None self.movePCBack = None self.outputTag = 1 self.originalAttributes = args[0] self.currentAttributes = args[1] self.repeatVariable = None self.tagContent = None self.localVarsDefined = 0 self.programCounter += 1 def cmdNoOp (self, command, args): self.programCounter += 1 def cmdUseMacro (self, command, args): """ args: (macroExpression, slotParams, endTagSymbol) Evaluates the expression, if it resolves to a SubTemplate it then places the slotParams into currentSlots and then jumps to the end tag """ value = self.context.evaluate (args[0], self.originalAttributes) if (value is None): # Don't output anything self.outputTag = 0 # Output none of our content or the existing content self.movePCForward = self.symbolTable [args[2]] self.programCounter += 1 return if (not value == simpleTALES.DEFAULTVALUE and isinstance (value, SubTemplate)): # We have a macro, so let's use it self.outputTag = 0 self.slotParameters = args[1] self.tagContent = (1, value) # NOTE: WE JUMP STRAIGHT TO THE END TAG, NO OTHER TAL/METAL COMMANDS ARE EVALUATED. self.programCounter = self.symbolTable [args[2]] return else: # Default, let's just run through as normal self.programCounter += 1 return def cmdDefineSlot (self, command, args): """ args: (slotName, endTagSymbol) If the slotName is filled then that is used, otherwise the original conent is used. """ if (args[0] in self.currentSlots): # This slot is filled, so replace us with that content self.outputTag = 0 self.tagContent = (1, self.currentSlots [args[0]]) # Output none of our content or the existing content # NOTE: NO FURTHER TAL/METAL COMMANDS ARE EVALUATED self.programCounter = self.symbolTable [args[1]] return # Slot isn't filled, so just use our own content self.programCounter += 1 return class HTMLTemplateInterpreter (TemplateInterpreter): def __init__ (self, minimizeBooleanAtts = 0): TemplateInterpreter.__init__ (self) self.minimizeBooleanAtts = minimizeBooleanAtts if (minimizeBooleanAtts): # Override the tagAsText method for this instance self.tagAsText = self.tagAsTextMinimizeAtts def tagAsTextMinimizeAtts (self, tagObj, singletonFlag=0): """ This returns a tag as text. """ tag,atts = tagObj result = ["<"] result.append (tag) upperTag = tag.upper() for attName, attValue in atts: if ('%s:%s' % (upperTag, attName.upper()) in HTML_BOOLEAN_ATTS): # We should output a minimised boolean value result.append (' ') result.append (attName) else: result.append (' ') result.append (attName) result.append ('="') result.append (html.escape (attValue, quote=1)) result.append ('"') if (singletonFlag): result.append (" />") else: result.append (">") return "".join (result) class Template: def __init__ (self, commands, macros, symbols, doctype = None): self.commandList = commands self.macros = macros self.symbolTable = symbols self.doctype = doctype self.log = logging.getLogger("simpleTAL.Template") # Setup the macros for macro in list(self.macros.values()): macro.setParentTemplate (self) # Setup the slots for cmnd, arg in self.commandList: if (cmnd == METAL_USE_MACRO): # Set the parent of each slot slotMap = arg[1] for slot in list(slotMap.values()): slot.setParentTemplate (self) def expand (self, context, outputFile, outputEncoding=None, interpreter=None): """ This method will write to the outputFile, using the encoding specified, the expanded version of this template. The context passed in is used to resolve all expressions with the template. """ # This method must wrap outputFile if required by the encoding, and write out # any template pre-amble (DTD, Encoding, etc) self.expandInline (context, outputFile, interpreter) def expandInline (self, context, outputFile, interpreter=None): """ Internally used when expanding a template that is part of a context.""" if (interpreter is None): ourInterpreter = TemplateInterpreter() ourInterpreter.initialise (context, outputFile) else: ourInterpreter = interpreter try: ourInterpreter.execute (self) except UnicodeError as unierror: self.log.error ("UnicodeError caused by placing a non-Unicode string in the Context object.") raise simpleTALES.ContextContentException ("Found non-unicode string in Context!") def getProgram (self): """ Returns a tuple of (commandList, startPoint, endPoint, symbolTable) """ return (self.commandList, 0, len (self.commandList), self.symbolTable) def __str__ (self): result = "Commands:\n" index = 0 for cmd in self.commandList: if (cmd[0] != METAL_USE_MACRO): result = result + "\n[%s] %s" % (str (index), str (cmd)) else: result = result + "\n[%s] %s, (%s{" % (str (index), str (cmd[0]), str (cmd[1][0])) for slot in list(cmd[1][1].keys()): result = result + "%s: %s" % (slot, str (cmd[1][1][slot])) result = result + "}, %s)" % str (cmd[1][2]) index += 1 result = result + "\n\nSymbols:\n" for symbol in list(self.symbolTable.keys()): result = result + "Symbol: " + str (symbol) + " points to: " + str (self.symbolTable[symbol]) + ", which is command: " + str (self.commandList[self.symbolTable[symbol]]) + "\n" result = result + "\n\nMacros:\n" for macro in list(self.macros.keys()): result = result + "Macro: " + str (macro) + " value of: " + str (self.macros[macro]) return result class SubTemplate (Template): """ A SubTemplate is part of another template, and is used for the METAL implementation. The two uses for this class are: 1 - metal:define-macro results in a SubTemplate that is the macro 2 - metal:fill-slot results in a SubTemplate that is a parameter to metal:use-macro """ def __init__ (self, startRange, endRangeSymbol): """ The parentTemplate is the template for which we are a sub-template. The startRange and endRange are indexes into the parent templates command list, and defines the range of commands that we can execute """ Template.__init__ (self, [], {}, {}) self.startRange = startRange self.endRangeSymbol = endRangeSymbol def setParentTemplate (self, parentTemplate): self.parentTemplate = parentTemplate self.commandList = parentTemplate.commandList self.symbolTable = parentTemplate.symbolTable def getProgram (self): """ Returns a tuple of (commandList, startPoint, endPoint, symbolTable) """ return (self.commandList, self.startRange, self.symbolTable[self.endRangeSymbol]+1, self.symbolTable) def __str__ (self): endRange = self.symbolTable [self.endRangeSymbol] result = "SubTemplate from %s to %s\n" % (str (self.startRange), str (endRange)) return result class HTMLTemplate (Template): """A specialised form of a template that knows how to output HTML """ def __init__ (self, commands, macros, symbols, doctype = None, minimizeBooleanAtts = 0): self.minimizeBooleanAtts = minimizeBooleanAtts Template.__init__ (self, commands, macros, symbols, doctype = None) def expand (self, context, outputFile, outputEncoding = "utf-8", interpreter=None): """ This method will write to the outputFile the expanded version of this template. The context passed in is used to resolve all expressions with the template. """ if (isinstance (outputFile, io.TextIOBase) or isinstance (outputFile, codecs.StreamWriter)): self.log.debug ("Text based output file detected") else: self.log.debug ("Bytes based output file detected - wrapping in codec for %s", outputEncoding) outputFile = codecs.lookup (outputEncoding).streamwriter (outputFile, 'xmlcharrefreplace') self.expandInline (context, outputFile, interpreter) def expandInline (self, context, outputFile, interpreter=None): """ Ensure we use the HTMLTemplateInterpreter""" if (interpreter is None): ourInterpreter = HTMLTemplateInterpreter(minimizeBooleanAtts = self.minimizeBooleanAtts) ourInterpreter.initialise (context, outputFile) else: ourInterpreter = interpreter Template.expandInline (self, context, outputFile, ourInterpreter) class XMLTemplate (Template): """A specialised form of a template that knows how to output XML """ def __init__ (self, commands, macros, symbols, doctype = None): Template.__init__ (self, commands, macros, symbols) self.doctype = doctype def expand (self, context, outputFile, outputEncoding = "utf-8", docType=None, suppressXMLDeclaration=False,interpreter=None): """ This method will write to the outputFile, using the encoding attached to the outputFile, the expanded version of this template. The context passed in is used to resolve all expressions with the template. """ if (isinstance (outputFile, io.TextIOBase) or isinstance (outputFile, codecs.StreamWriter)): self.log.debug ("Text based output file detected") else: self.log.debug ("Bytes based output file detected - wrapping in codec for %s", outputEncoding) outputFile = codecs.lookup (outputEncoding).streamwriter (outputFile, 'xmlcharrefreplace') if (not suppressXMLDeclaration): if (outputEncoding.lower() != "utf-8"): outputFile.write ('\n' % outputEncoding.lower()) else: outputFile.write ('\n') if not docType and self.doctype: docType = self.doctype if docType: outputFile.write (docType) outputFile.write ('\n') self.expandInline (context, outputFile, interpreter) class TemplateCompiler: def __init__ (self): """ Initialise a template compiler. """ self.commandList = [] self.tagStack = [] self.symbolLocationTable = {} self.macroMap = {} self.endTagSymbol = 1 self.commandHandler = {} self.commandHandler [TAL_DEFINE] = self.compileCmdDefine self.commandHandler [TAL_CONDITION] = self.compileCmdCondition self.commandHandler [TAL_REPEAT] = self.compileCmdRepeat self.commandHandler [TAL_CONTENT] = self.compileCmdContent self.commandHandler [TAL_REPLACE] = self.compileCmdReplace self.commandHandler [TAL_ATTRIBUTES] = self.compileCmdAttributes self.commandHandler [TAL_OMITTAG] = self.compileCmdOmitTag # Metal commands self.commandHandler [METAL_USE_MACRO] = self.compileMetalUseMacro self.commandHandler [METAL_DEFINE_SLOT] = self.compileMetalDefineSlot self.commandHandler [METAL_FILL_SLOT] = self.compileMetalFillSlot self.commandHandler [METAL_DEFINE_MACRO] = self.compileMetalDefineMacro # Default namespaces self.setTALPrefix ('tal') self.tal_namespace_prefix_stack = [] self.metal_namespace_prefix_stack = [] self.tal_namespace_prefix_stack.append ('tal') self.setMETALPrefix ('metal') self.metal_namespace_prefix_stack.append ('metal') self.log = logging.getLogger ("simpleTAL.TemplateCompiler") def setTALPrefix (self, prefix): self.tal_namespace_prefix = prefix self.tal_namespace_omittag = '%s:omit-tag' % self.tal_namespace_prefix self.tal_attribute_map = {} self.tal_attribute_map ['%s:attributes'%prefix] = TAL_ATTRIBUTES self.tal_attribute_map ['%s:content'%prefix]= TAL_CONTENT self.tal_attribute_map ['%s:define'%prefix] = TAL_DEFINE self.tal_attribute_map ['%s:replace'%prefix] = TAL_REPLACE self.tal_attribute_map ['%s:omit-tag'%prefix] = TAL_OMITTAG self.tal_attribute_map ['%s:condition'%prefix] = TAL_CONDITION self.tal_attribute_map ['%s:repeat'%prefix] = TAL_REPEAT def setMETALPrefix (self, prefix): self.metal_namespace_prefix = prefix self.metal_attribute_map = {} self.metal_attribute_map ['%s:define-macro'%prefix] = METAL_DEFINE_MACRO self.metal_attribute_map ['%s:use-macro'%prefix] = METAL_USE_MACRO self.metal_attribute_map ['%s:define-slot'%prefix] = METAL_DEFINE_SLOT self.metal_attribute_map ['%s:fill-slot'%prefix] = METAL_FILL_SLOT def popTALNamespace (self): newPrefix = self.tal_namespace_prefix_stack.pop() self.setTALPrefix (newPrefix) def popMETALNamespace (self): newPrefix = self.metal_namespace_prefix_stack.pop() self.setMETALPrefix (newPrefix) def tagAsText (self, tagObj, singletonFlag=0): """ This returns a tag as text. """ tag,atts = tagObj result = ["<"] result.append (tag) for attName, attValue in atts: result.append (' ') result.append (attName) result.append ('="') result.append (html.escape (attValue, quote=1)) result.append ('"') if (singletonFlag): result.append (" />") else: result.append (">") return "".join (result) def getTemplate (self): template = Template (self.commandList, self.macroMap, self.symbolLocationTable) return template def addCommand (self, command): if (command[0] == TAL_OUTPUT and (len (self.commandList) > 0) and self.commandList[-1][0] == TAL_OUTPUT): # We can combine output commands self.commandList[-1] = (TAL_OUTPUT, self.commandList[-1][1] + command[1]) else: self.commandList.append (command) def addTag (self, tag, tagProperties={}): """ Used to add a tag to the stack. Various properties can be passed in the dictionary as being information required by the tag. Currently supported properties are: 'command' - The (command,args) tuple associated with this command 'originalAtts' - The original attributes that include any metal/tal attributes 'endTagSymbol' - The symbol associated with the end tag for this element 'popFunctionList' - A list of functions to execute when this tag is popped 'singletonTag' - A boolean to indicate that this is a singleton flag """ # Add the tag to the tagStack (list of tuples (tag, properties, useMacroLocation)) self.log.debug ("Adding tag %s to stack" % tag[0]) command = tagProperties.get ('command',None) originalAtts = tagProperties.get ('originalAtts', None) singletonTag = tagProperties.get ('singletonTag', 0) if (command is not None): if (command[0] == METAL_USE_MACRO): self.tagStack.append ((tag, tagProperties, len (self.commandList)+1)) else: self.tagStack.append ((tag, tagProperties, None)) else: self.tagStack.append ((tag, tagProperties, None)) if (command is not None): # All tags that have a TAL attribute on them start with a 'start scope' self.addCommand((TAL_START_SCOPE, (originalAtts, tag[1]))) # Now we add the TAL command self.addCommand(command) else: # It's just a straight output, so create an output command and append it self.addCommand((TAL_OUTPUT, self.tagAsText (tag, singletonTag))) def popTag (self, tag, omitTagFlag=0): """ omitTagFlag is used to control whether the end tag should be included in the output or not. In HTML 4.01 there are several tags which should never have end tags, this flag allows the template compiler to specify that these should not be output. """ while (len (self.tagStack) > 0): oldTag, tagProperties, useMacroLocation = self.tagStack.pop() endTagSymbol = tagProperties.get ('endTagSymbol', None) popCommandList = tagProperties.get ('popFunctionList', []) singletonTag = tagProperties.get ('singletonTag', 0) for func in popCommandList: func() self.log.debug ("Popped tag %s off stack" % oldTag[0]) if (oldTag[0] == tag[0]): # We've found the right tag, now check to see if we have any TAL commands on it if (endTagSymbol is not None): # We have a command (it's a TAL tag) # Note where the end tag symbol should point (i.e. the next command) self.symbolLocationTable [endTagSymbol] = len (self.commandList) # We need a "close scope and tag" command self.addCommand((TAL_ENDTAG_ENDSCOPE, (tag[0], omitTagFlag, singletonTag))) return elif (omitTagFlag == 0 and singletonTag == 0): # We are popping off an un-interesting tag, just add the close as text self.addCommand((TAL_OUTPUT, '')) return else: # We are suppressing the output of this tag, so just return return else: # We have a different tag, which means something like
which never closes is in # between us and the real tag. # If the tag that we did pop off has a command though it means un-balanced TAL tags! if (endTagSymbol is not None): # ERROR msg = "TAL/METAL Elements must be balanced - found close tag %s expecting %s" % (tag[0], oldTag[0]) self.log.error (msg) raise TemplateParseException (self.tagAsText(oldTag), msg) self.log.error ("Close tag %s found with no corresponding open tag." % tag[0]) raise TemplateParseException ("" % tag[0], "Close tag encountered with no corresponding open tag.") def parseStartTag (self, tag, attributes, singletonElement=0): # Note down the tag we are handling, it will be used for error handling during # compilation self.currentStartTag = (tag, attributes) # Look for tal/metal attributes foundTALAtts = [] foundMETALAtts = [] foundCommandsArgs = {} cleanAttributes = [] originalAttributes = {} tagProperties = {} popTagFuncList = [] TALElementNameSpace = 0 prefixToAdd = "" tagProperties ['singletonTag'] = singletonElement # Determine whether this element is in either the METAL or TAL namespace if (tag.find (':') > 0): # We have a namespace involved, so let's look to see if its one of ours namespace = tag[0:tag.find (':')] if (namespace == self.metal_namespace_prefix): TALElementNameSpace = 1 prefixToAdd = self.metal_namespace_prefix +":" elif (namespace == self.tal_namespace_prefix): TALElementNameSpace = 1 prefixToAdd = self.tal_namespace_prefix +":" if (TALElementNameSpace): # We should treat this an implicit omit-tag foundTALAtts.append (TAL_OMITTAG) # Will go to default, i.e. yes foundCommandsArgs [TAL_OMITTAG] = "" for att, value in attributes: originalAttributes [att] = value if (TALElementNameSpace and not att.find (':') > 0): # This means that the attribute name does not have a namespace, so use the prefix for this tag. commandAttName = prefixToAdd + att else: commandAttName = att self.log.debug ("Command name is now %s" % commandAttName) if (att[0:5] == "xmlns"): # We have a namespace declaration. prefix = att[6:] if (value == METAL_NAME_URI): # It's a METAL namespace declaration if (len (prefix) > 0): self.metal_namespace_prefix_stack.append (self.metal_namespace_prefix) self.setMETALPrefix (prefix) # We want this function called when the scope ends popTagFuncList.append (self.popMETALNamespace) else: # We don't allow METAL/TAL to be declared as a default msg = "Can not use METAL name space by default, a prefix must be provided." raise TemplateParseException (self.tagAsText (self.currentStartTag), msg) elif (value == TAL_NAME_URI): # TAL this time if (len (prefix) > 0): self.tal_namespace_prefix_stack.append (self.tal_namespace_prefix) self.setTALPrefix (prefix) # We want this function called when the scope ends popTagFuncList.append (self.popTALNamespace) else: # We don't allow METAL/TAL to be declared as a default msg = "Can not use TAL name space by default, a prefix must be provided." raise TemplateParseException (self.tagAsText (self.currentStartTag), msg) else: # It's nothing special, just an ordinary namespace declaration cleanAttributes.append ((att, value)) elif (commandAttName in self.tal_attribute_map): # It's a TAL attribute cmnd = self.tal_attribute_map [commandAttName] if (cmnd == TAL_OMITTAG and TALElementNameSpace): self.log.warning ("Supressing omit-tag command present on TAL or METAL element") else: foundCommandsArgs [cmnd] = value foundTALAtts.append (cmnd) elif (commandAttName in self.metal_attribute_map): # It's a METAL attribute cmnd = self.metal_attribute_map [commandAttName] foundCommandsArgs [cmnd] = value foundMETALAtts.append (cmnd) else: cleanAttributes.append ((att, value)) tagProperties ['popFunctionList'] = popTagFuncList # This might be just content if ((len (foundTALAtts) + len (foundMETALAtts)) == 0): # Just content, add it to the various stacks self.addTag ((tag, cleanAttributes), tagProperties) return # Create a symbol for the end of the tag - we don't know what the offset is yet self.endTagSymbol += 1 tagProperties ['endTagSymbol'] = self.endTagSymbol # Sort the METAL commands foundMETALAtts.sort() # Sort the tags by priority foundTALAtts.sort() # We handle the METAL before the TAL allCommands = foundMETALAtts + foundTALAtts firstTag = 1 for talAtt in allCommands: # Parse and create a command for each cmnd = self.commandHandler [talAtt](foundCommandsArgs[talAtt]) if (cmnd is not None): if (firstTag): # The first one needs to add the tag firstTag = 0 tagProperties ['originalAtts'] = originalAttributes tagProperties ['command'] = cmnd self.addTag ((tag, cleanAttributes), tagProperties) else: # All others just append self.addCommand(cmnd) if (firstTag): tagProperties ['originalAtts'] = originalAttributes tagProperties ['command'] = (TAL_STARTTAG, (tag, singletonElement)) self.addTag ((tag, cleanAttributes), tagProperties) else: # Add the start tag command in as a child of the last TAL command self.addCommand((TAL_STARTTAG, (tag,singletonElement))) def parseEndTag (self, tag): """ Just pop the tag and related commands off the stack. """ self.popTag ((tag,None)) def parseData (self, data): # Just add it as an output self.addCommand((TAL_OUTPUT, data)) def compileCmdDefine (self, argument): # Compile a define command, resulting argument is: # [(isLocalFlag (Y/n), variableName, variablePath),...] # Break up the list of defines first commandArgs = [] # We only want to match semi-colons that are not escaped argumentSplitter = re.compile ('(?2 elements means a local|global flag if (len (stmtBits) > 2): if (stmtBits[0] == 'global'): isLocal = 0 varName = stmtBits[1] expression = ' '.join (stmtBits[2:]) elif (stmtBits[0] == 'local'): varName = stmtBits[1] expression = ' '.join (stmtBits[2:]) else: # Must be a space in the expression that caused the >3 thing varName = stmtBits[0] expression = ' '.join (stmtBits[1:]) else: # Only two bits varName = stmtBits[0] expression = ' '.join (stmtBits[1:]) commandArgs.append ((isLocal, varName, expression)) return (TAL_DEFINE, commandArgs) def compileCmdCondition (self, argument): # Compile a condition command, resulting argument is: # path, endTagSymbol # Sanity check if (len (argument) == 0): # No argument passed msg = "No argument passed! condition commands must be of the form: 'path'" self.log.error (msg) raise TemplateParseException (self.tagAsText (self.currentStartTag), msg) return (TAL_CONDITION, (argument, self.endTagSymbol)) def compileCmdRepeat (self, argument): # Compile a repeat command, resulting argument is: # (varname, expression, endTagSymbol) attProps = argument.split (' ') if (len (attProps) < 2): # Error, badly formed repeat command msg = "Badly formed repeat command '%s'. Repeat commands must be of the form: 'localVariable path'" % argument self.log.error (msg) raise TemplateParseException (self.tagAsText (self.currentStartTag), msg) varName = attProps [0] expression = " ".join (attProps[1:]) return (TAL_REPEAT, (varName, expression, self.endTagSymbol)) def compileCmdContent (self, argument, replaceFlag=0): # Compile a content command, resulting argument is # (replaceFlag, structureFlag, expression, endTagSymbol) # Sanity check if (len (argument) == 0): # No argument passed msg = "No argument passed! content/replace commands must be of the form: 'path'" self.log.error (msg) raise TemplateParseException (self.tagAsText (self.currentStartTag), msg) structureFlag = 0 attProps = argument.split (' ') if (len(attProps) > 1): if (attProps[0] == "structure"): structureFlag = 1 express = " ".join (attProps[1:]) elif (attProps[1] == "text"): structureFlag = 0 express = " ".join (attProps[1:]) else: # It's not a type selection after all - assume it's part of the path express = argument else: express = argument return (TAL_CONTENT, (replaceFlag, structureFlag, express, self.endTagSymbol)) def compileCmdReplace (self, argument): return self.compileCmdContent (argument, replaceFlag=1) def compileCmdAttributes (self, argument): # Compile tal:attributes into attribute command # Argument: [(attributeName, expression)] # Break up the list of attribute settings first commandArgs = [] # We only want to match semi-colons that are not escaped argumentSplitter = re.compile ('(?") else: result.append (">") return "".join (result) def handle_startendtag (self, tag, attributes): self.handle_starttag (tag, attributes) if not (tag.upper() in HTML_FORBIDDEN_ENDTAG): self.handle_endtag(tag) def handle_starttag (self, tag, attributes): self.log.debug ("Received Start Tag: " + tag + " Attributes: " + str (attributes)) atts = [] for att, attValue in attributes: # We need to spot empty tal:omit-tags if (attValue is None): if (att == self.tal_namespace_omittag): atts.append ((att, "")) else: atts.append ((att, att)) elif not HTML_ENTITIES_PRE_EXPANDED: # Expand any SGML entity references or char references goodAttValue = [] last = 0 match = ENTITY_REF_REGEX.search (attValue) while (match): goodAttValue.append (attValue[last:match.start()]) ref = attValue[match.start():match.end()] if (ref.startswith ('&#')): # A char reference if (ref[2] in ['x', 'X']): # Hex refValue = int (ref[3:-1], 16) else: refValue = int (ref[2:-1]) goodAttValue.append (chr (refValue)) else: # A named reference. goodAttValue.append (chr (sgmlentitynames.htmlNameToUnicodeNumber.get (ref[1:-1], 65533))) last = match.end() match = ENTITY_REF_REGEX.search (attValue, last) goodAttValue.append (attValue [last:]) atts.append ((att, "".join (goodAttValue))) else: atts.append ((att, attValue)) if (tag.upper() in HTML_FORBIDDEN_ENDTAG): # This should have no end tag, so we just do the start and suppress the end self.parseStartTag (tag, atts) self.log.debug ("End tag forbidden, generating close tag with no output.") self.popTag ((tag, None), omitTagFlag=1) else: self.parseStartTag (tag, atts) def handle_endtag (self, tag): self.log.debug ("Recieved End Tag: " + tag) if (tag.upper() in HTML_FORBIDDEN_ENDTAG): self.log.warning ("HTML 4.01 forbids end tags for the %s element" % tag) else: # Normal end tag self.popTag ((tag, None)) def handle_data (self, data): self.parseData (html.escape (data, quote=False)) # These two methods are required so that we expand all character and entity references prior to parsing the template. def handle_charref (self, ref): self.log.debug ("Got Ref: %s", ref) self.parseData (chr (int (ref))) def handle_entityref (self, ref): self.log.debug ("Got Ref: %s", ref) # Use handle_data so that <&> are re-encoded as required. self.handle_data( chr (sgmlentitynames.htmlNameToUnicodeNumber.get (ref, 65533))) # Handle document type declarations def handle_decl (self, data): self.parseData ('' % data) # Pass comments through un-affected. def handle_comment (self, data): self.parseData ('' % data) def handle_pi (self, data): self.log.debug ("Recieved processing instruction.") self.parseData ('' % data) def report_unbalanced (self, tag): self.log.warning ("End tag %s present with no corresponding open tag.") def getTemplate (self): template = HTMLTemplate (self.commandList, self.macroMap, self.symbolLocationTable, minimizeBooleanAtts = self.minimizeBooleanAtts) return template class XMLTemplateCompiler (TemplateCompiler, xml.sax.handler.ContentHandler, xml.sax.handler.DTDHandler, LexicalHandler): def __init__ (self): TemplateCompiler.__init__ (self) xml.sax.handler.ContentHandler.__init__ (self) self.doctype = None self.log = logging.getLogger ("simpleTAL.XMLTemplateCompiler") self.singletonElement = 0 def parseTemplate (self, templateFile): self.ourParser = xml.sax.make_parser() self.log.debug ("Setting features of parser") try: self.ourParser.setFeature (xml.sax.handler.feature_external_ges, False) self.log.debug ("Set SAX Handler feature OK") except: pass if use_lexical_handler: self.ourParser.setProperty(xml.sax.handler.property_lexical_handler, self) self.ourParser.setContentHandler (self) self.ourParser.setDTDHandler (self) self.ourParser.parse (templateFile) def startDTD(self, name, public_id, system_id): self.log.debug ("Recieved DOCTYPE: " + name + " public_id: " + public_id + " system_id: " + system_id) if public_id: self.doctype = '' % (name, public_id, system_id,) else: self.doctype = '' % (name, system_id,) def startElement (self, tag, attributes): self.log.debug ("Recieved Real Start Tag: " + tag + " Attributes: " + str (attributes)) try: xmlText = self.ourParser.getProperty (xml.sax.handler.property_xml_string) if (isinstance (xmlText, str)): # Once issue 6686 is solved this should allow singletons to be detected if (SINGLETON_XML_REGEX.match (xmlText)): # This is a singleton! self.singletonElement=1 else: if (SINGLETON_BYTES_XML_REGEX.match (xmlText)): # This is a singleton! self.singletonElement = 1 except xml.sax.SAXException as e: # Parser doesn't support this property pass # Convert attributes into a list of tuples atts = [] for att in attributes.getNames(): self.log.debug ("Attribute name %s has value %s" % (att, attributes[att])) atts.append ((att, attributes [att])) self.parseStartTag (tag, atts, singletonElement=self.singletonElement) def endElement (self, tag): self.log.debug ("Recieved Real End Tag: " + tag) self.parseEndTag (tag) self.singletonElement = 0 def skippedEntity (self, name): self.log.info ("Recieved skipped entity: %s" % name) self.characters( chr (sgmlentitynames.htmlNameToUnicodeNumber.get (name, 65533))) def characters (self, data): #self.log.debug ("Recieved Real Data: " + data) # Escape any data we recieve - we don't want any: <&> in there. self.parseData (html.escape (data, quote=False)) def processingInstruction (self, target, data): self.log.debug ("Recieved processing instruction.") self.parseData ('' % (target, data)) def comment (self, data): # This is only called if your XML parser supports the LexicalHandler interface. self.parseData ('' % data) def getTemplate (self): template = XMLTemplate (self.commandList, self.macroMap, self.symbolLocationTable, self.doctype) return template def compileHTMLTemplate (template, minimizeBooleanAtts = 0): """ Reads the templateFile and produces a compiled template. To use the resulting template object call: template.expand (context, outputFile) """ if (isinstance (template, str)): # It's a string! templateFile = io.StringIO (template) else: templateFile = template compiler = HTMLTemplateCompiler() compiler.parseTemplate (templateFile, minimizeBooleanAtts) return compiler.getTemplate() def compileXMLTemplate (template): """ Reads the templateFile and produces a compiled template. To use the resulting template object call: template.expand (context, outputFile) """ if (isinstance (template, str)): # It's a string! templateFile = io.StringIO (template) elif (isinstance (template, bytes)): templateFile = io.BytesIO (template) else: templateFile = template compiler = XMLTemplateCompiler() compiler.parseTemplate (templateFile) return compiler.getTemplate()