""" Some common declarations for the xmlproc system gathered in one file. """ import string,re,urlparse,os from xmlapp import * import errors # Standard exceptions class OutOfDataException(Exception): """An exception that signals that more data is expected, but the current buffer has been exhausted.""" pass # ============================== # The general entity parser # ============================== class EntityParser: """A generalized parser for XML entities, whether DTD, documents or even catalog files.""" def __init__(self): # --- Creating support objects self.err=ErrorHandler(self) self.ent=EntityHandler(self.err) self.isf=InputSourceFactory() self.pubres=PubIdResolver() self.data_charset="iso-8859-1" self.errors=errors.get_error_list("en") self.reset() def set_error_language(self,language): """Sets the language in which errors are reported. (ISO 3166 codes.) Throws a KeyError if the language is not supported.""" self.errors=errors.get_error_list(string.lower(language)) def set_error_handler(self,err): "Sets the object to send error events to." self.err=err def set_pubid_resolver(self,pubres): self.pubres=pubres def set_entity_handler(self,ent): "Sets the object that resolves entity references." self.ent=ent def set_inputsource_factory(self,isf): "Sets the object factory used to create input sources from sysids." self.isf=isf def set_data_charset(self,charset): """Tells the parser which character encoding to use when reporting data to applications.""" self.data_charset=charset def parse_resource(self,sysID,bufsize=16384): """Begin parsing an XML entity with the specified public and system identifiers (the system identifier, a URI, is required). Only used for the document entity, not to handle subentities, which open_entity takes care of.""" self.current_sysID=sysID try: infile=self.isf.create_input_source(sysID) except IOError,e: self.report_error(3000,sysID) return self.read_from(infile,bufsize) infile.close() self.flush() self.parseEnd() def open_entity(self,sysID): """Starts parsing a new entity, pushing the old onto the stack. This method must not be used to start parsing, use parse_resource for that.""" sysID=join_sysids(self.get_current_sysid(),sysID) try: inf=self.isf.create_input_source(sysID) except IOError,e: self.report_error(3000,sysID) return self.ent_stack.append(self.get_current_sysid(),self.data,self.pos,\ self.line,self.last_break,self.datasize,\ self.last_upd_pos,self.block_offset) self.current_sysID=sysID self.pos=0 self.line=1 self.last_break=0 self.data="" self.read_from(inf) self.flush() self.pop_entity() def push_entity(self,sysID,contents): """Parse some text and consider it a new entity, making it possible to return to the original entity later.""" self.ent_stack.append(self.get_current_sysid(),self.data,self.pos,\ self.line,self.last_break,self.datasize,\ self.last_upd_pos,self.block_offset) self.data=contents self.current_sysID=sysID self.pos=0 self.line=1 self.last_break=0 self.datasize=len(contents) self.last_upd_pos=0 def pop_entity(self): "Skips out of the current entity and back to the previous one." if self.ent_stack==[]: self.report_error(4000) (self.current_sysID,self.data,self.pos,self.line,self.last_break,\ self.datasize,self.last_upd_pos,self.block_offset)=self.ent_stack[-1] del self.ent_stack[-1] self.final=0 def read_from(self,fileobj,bufsize=16384): """Reads data from a file-like object until EOF. Does not close it. **WARNING**: This method does not call the parseStart/parseEnd methods, since it does not know if it may be called several times. Use parse_resource if you just want to read a file.""" while 1: buf=fileobj.read(bufsize) if buf=="": break try: self.feed(buf) except OutOfDataException,e: break def reset(self): """Resets the parser, losing all unprocessed data.""" self.ent_stack=[] self.open_ents=[] # Used to test for entity recursion self.current_sysID="Unknown" self.first_feed=1 # Block information self.data="" self.final=0 self.datasize=0 self.start_point=-1 # Location tracking self.line=1 self.last_break=0 self.block_offset=0 # Offset from start of stream to start of cur block self.pos=0 self.last_upd_pos=0 def feed(self,new_data): """Accepts more data from the data source. This method must set self.datasize and correctly update self.pos and self.data.""" if self.first_feed: self.first_feed=0 self.parseStart() self.update_pos() # Update line/col count if self.start_point==-1: self.block_offset=self.block_offset+self.datasize self.data=self.data[self.pos:] self.last_break=self.last_break-self.pos # Keep track of column self.pos=0 self.last_upd_pos=0 # Adding new data and doing line end normalization self.data=string.replace(self.data+new_data, "\015\012","\012") self.datasize=len(self.data) self.do_parse() def close(self): "Closes the parser, processing all remaining data. Calls parseEnd." self.flush() self.parseEnd() def parseStart(self): "Called before the parse starts to notify subclasses." pass def parseEnd(self): "Called when there are no more data to notify subclasses." pass def flush(self): "Parses any remnants of data in the last block." if not self.pos+1==self.datasize: self.final=1 try: self.do_parse() except OutOfDataException,e: self.report_error(3001) # --- GENERAL UTILITY # --- LOW-LEVEL SCANNING METHODS def set_start_point(self): """Stores the current position and tells the parser not to forget any of the data beyond this point until get_region is called.""" self.start_point=self.pos def store_state(self): """Makes the parser remember where we are now, so we can go back later with restore_state.""" self.set_start_point() self.old_state=(self.last_upd_pos,self.line,self.last_break) def restore_state(self): """Goes back to a state previously remembered with store_state.""" self.pos=self.start_point self.start_point=-1 (self.last_upd_pos,self.line,self.last_break)=self.old_state def get_region(self): """Returns the area from start_point to current position and remove start_point.""" data=self.data[self.start_point:self.pos] self.start_point=-1 return data def find_reg(self,regexp,required=1): """Moves self.pos to the first character that matches the regexp and returns everything from pos and up to (but not including) that character.""" oldpos=self.pos mo=regexp.search(self.data,self.pos) if mo==None: if self.final and not required: self.pos=len(self.data) # Just moved to the end return self.data[oldpos:] raise OutOfDataException() self.pos=mo.start(0) return self.data[oldpos:self.pos] def scan_to(self,target): "Moves self.pos to beyond target and returns skipped text." new_pos=string.find(self.data,target,self.pos) if new_pos==-1: raise OutOfDataException() res=self.data[self.pos:new_pos] self.pos=new_pos+len(target) return res def get_index(self,target): "Finds the position where target starts and returns it." new_pos=string.find(self.data,target,self.pos) if new_pos==-1: raise OutOfDataException() return new_pos def test_str(self,test_str): "See if text at current position matches test_str, without moving." if self.datasize-self.posself.datasize-5 and not self.final: raise OutOfDataException() return regexp.match(self.data,self.pos)!=None def get_match(self,regexp): "Returns the result of matching the regexp and advances self.pos." if self.pos>self.datasize-5 and not self.final: raise OutOfDataException() ent=regexp.match(self.data,self.pos) if ent==None: self.report_error(reg2code[regexp.pattern]) return "" end=ent.end(0) # Speeds us up slightly if end==self.datasize: raise OutOfDataException() self.pos=end return ent.group(0) def update_pos(self): "Updates (line,col)-pos by checking processed blocks." breaks=string.count(self.data,"\n",self.last_upd_pos,self.pos) self.last_upd_pos=self.pos if breaks>0: self.line=self.line+breaks self.last_break=string.rfind(self.data,"\n",0,self.pos) def get_wrapped_match(self,wraps): "Returns a contained match. Useful for regexps inside quotes." found=0 for (wrap,regexp) in wraps: if self.test_str(wrap): found=1 self.pos=self.pos+len(wrap) break if not found: msg="" for (wrap,regexp) in wraps[:-1]: msg="%s'%s', " % (msg,wrap) self.report_error(3004,(msg[:-2],wraps[-1][0])) data=self.get_match(regexp) if not self.now_at(wrap): self.report_error(3005,wrap) return data #--- ERROR HANDLING def report_error(self,number,args=None): try: msg=self.errors[number] if args!=None: msg=msg % args except KeyError: msg=self.errors[4002] # Unknown err msg :-) if number<2000: self.err.warning(msg) elif number<3000: self.err.error(msg) else: # Make parser stop reporting data events! self.err.fatal(msg) #--- USEFUL METHODS def get_current_sysid(self): "Returns the sysid of the file we are reading now." return self.current_sysID def set_sysid(self,sysID): "Sets the current system identifier. Does not store the old one." self.current_sysID=sysID def get_offset(self): "Returns the current offset from the start of the stream." return self.block_offset+self.pos def get_line(self): "Returns the current line number." self.update_pos() return self.line def get_column(self): "Returns the current column position." self.update_pos() return self.pos-self.last_break def is_root_entity(self): "Returns true if the current entity is the root entity." return self.ent_stack==[] def is_external(self): """Returns true if the current entity is an external entity. The root (or document) entity is not considered external.""" return self.ent_stack!=[] and \ self.ent_stack[0][0]!=self.get_current_sysid() # --- A collection of useful functions # Utility functions def unhex(hex_value): "Converts a string hex-value to an integer." sum=0 for char in hex_value: sum=sum*16 char=ord(char) if char<58 and char>=48: sum=sum+(char-48) elif char>=97 and char<=102: sum=sum+(char-87) elif char>=65 and char<=70: sum=sum+(char-55) # else ERROR, but it can't occur here return sum def matches(regexp,str): mo=regexp.match(str) return mo!=None and len(mo.group(0))==len(str) def join_sysids(base,url): if urlparse.urlparse(base)[0]=="": return os.path.join(os.path.split(base)[0],url) else: return urlparse.urljoin(base,url) # --- Some useful regexps namestart="ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz_:" namechars=namestart+"0123456789.-" whitespace="\n\t \r" reg_ws=re.compile("[\n\t \r]+") reg_ver=re.compile("[-a-zA-Z0-9_.:]+") reg_enc_name=re.compile("[A-Za-z][-A-Za-z0-9._]*") reg_std_alone=re.compile("yes|no") reg_comment_content=re.compile("([^-]|-[^-])*") reg_name=re.compile("[A-Za-z_:][\-A-Za-z_:.0-9]*") reg_names=re.compile("[A-Za-z_:][\-A-Za-z_:.0-9]*" "([\n\t \r]+[A-Za-z_:][\-A-Za-z_:.0-9]*)*") reg_nmtoken=re.compile("[\-A-Za-z_:.0-9]+") reg_nmtokens=re.compile("[\-A-Za-z_:.0-9]+([\n\t \r]+[\-A-Za-z_:.0-9]+)*") reg_sysid_quote=re.compile("[^\"]*") reg_sysid_apo=re.compile("[^']*") reg_pubid_quote=re.compile("[- \n\t\ra-zA-Z0-9'()+,./:=?;!*#@$_%]*") reg_pubid_apo=re.compile("[- \n\t\ra-zA-Z0-9()+,./:=?;!*#@$_%]*") reg_start_tag=re.compile("<[A-Za-z_:]") reg_quoted_attr=re.compile("[^<\"]*") reg_apo_attr=re.compile("[^<']*") reg_c_data=re.compile("[<&]") reg_pe_ref=re.compile("%[A-Za-z_:][\-A-Za-z_:.0-9]*;") reg_ent_val_quote=re.compile("[^\"]+") reg_ent_val_apo=re.compile("[^\']+") reg_attr_type=re.compile(r"CDATA|IDREFS|IDREF|ID|ENTITY|ENTITIES|NMTOKENS|" "NMTOKEN") reg_attr_def=re.compile(r"#REQUIRED|#IMPLIED") reg_digits=re.compile("[0-9]+") reg_hex_digits=re.compile("[0-9a-fA-F]+") reg_res_pi=re.compile("xml",re.I) reg_int_dtd=re.compile("\"|'|<\\?|