### URL ############################################################################################## # Code for identifying, parsing and retrieving URL's. # The asynchronous URLAccumulator is subclassed in all other services. # Author: Tom De Smedt. # Copyright (c) 2007 by Tom De Smedt. # See LICENSE.txt for details. import os import socket, urllib, urllib2, urlparse import thread, time from warnings import warn from cache import Cache ### SETTINGS ######################################################################################### USER_AGENT = "NodeBox/1.9.4 +http://nodebox.net" REFERER = "http://nodebox.net" ### URLERROR ######################################################################################### class URLError(Exception): # A fault in the URL, like a missing t in htp:// def __str__(self): return str(self.__class__) class URLTimeout(URLError): # URL took to long to load. def __str__(self): return str(self.__class__) class HTTPError(URLError): # Error on server. def __str__(self): return str(self.__class__) class HTTP401Authentication(HTTPError): # URL requires a login and password. def __str__(self): return str(self.__class__) class HTTP403Forbidden(HTTPError): # No access to this URL (user-agent?) def __str__(self): return str(self.__class__) class HTTP404NotFound(HTTPError): # URL doesn't exist on the internet. def __str__(self): return str(self.__class__) ### URLPARSER ######################################################################################## class URLParser: def __init__(self, url="", method="get"): """ Splits an url string into different parts. The parts are: protocol, domain, login, username, password, port, path, page, query, anchor. The method defaults to get when the url has a query part. Setting it to post will submit the query by POST when opening the url. """ # If the url is a URLParser, copy POST parameters correctly. is_post_urlparser = False if isinstance(url, URLParser) and url.method == "post": is_post_urlparser = True url.method = "get" # If the url is a URLParser, use its string representation. # See that the original object's method is correctly reset. urlstr = str(url) if is_post_urlparser: url.method = "post" url = urlstr # Consider the following url: # http://user:pass@example.com:992/animal/bird?species=seagull#wings # protocol: http # domain: example.com url = urlparse.urlsplit(url) self.protocol = url[0] self.domain = url[1] # username: user # password: pass self.username = "" self.password = "" if self.domain.find("@") >= 0: login = self.domain.split("@")[0] if login.find(":") >= 0: self.username = login.split(":")[0] self.password = login.split(":")[1] self.domain = self.domain.split("@")[1] # port: 992 self.port = "" if self.domain.find(":") >= 0: p = self.domain.split(":") if p[1].isdigit(): self.port = p[1] self.domain = p[0] # path: /animal/ # page: bird self.path = url[2] self.page = "" if not self.path.endswith("/"): if self.path.find("/") >= 0: self.page = self.path.split("/")[-1] self.path = self.path[:-len(self.page)] else: self.page = self.path self.path = "" self.filename = self.page # query: {"species": "seagull"} self.query = {} self.method = method if url[3] != "": self.method = "get" if is_post_urlparser: self.method = "post" for param in url[3].split("&"): key, value = "", "" if param.find("=") >= 0: try: (key, value) = param.split("=") except: key = param else: key = param if key != "": self.query[key] = value # anchor: wings self.anchor = url[4] def __str__(self): """ Reforms a url string from the different parts. """ url = "" if self.protocol != "" : url += self.protocol + "://" if self.username != "" : url += self.username + ":" + self.password + "@" if self.domain != "" : url += self.domain if self.port != "" : url += ":" + self.port if self.path != "" : url += self.path if self.page != "" : url += self.page if self.method == "get" and \ len(self.query) > 0 : url += "?" + urllib.urlencode(self.query) if self.anchor != "" : url += "#" + self.anchor return url def _address(self): return str(self) address = property(_address) def parse(url): return URLParser(url) def create(url="", method="get"): return URLParser(url, method) #url = parse("http://user:pass@example.com:992/animal/bird?species=seagull#wings") #print url.domain #print url.path #print url.page #print url ### URL OPENER ####################################################################################### PROXY = None def set_proxy(host, type="https"): global PROXY if host != None: PROXY = (host, type) else: PROXY = None def open(url, wait=10): """ Returns a connection to a url which you can read(). When the wait amount is exceeded, raises a URLTimeout. When an error occurs, raises a URLError. 404 errors specifically return a HTTP404NotFound. """ # If the url is a URLParser, get any POST parameters. post = None if isinstance(url, URLParser) and url.method == "post": post = urllib.urlencode(url.query) # If the url is a URLParser (or a YahooResult or something), # use its string representation. url = str(url) # Use urllib instead of urllib2 for local files. if os.path.exists(url): return urllib.urlopen(url) else: socket.setdefaulttimeout(wait) try: #connection = urllib2.urlopen(url, post) request = urllib2.Request(url, post, {"User-Agent": USER_AGENT, "Referer": REFERER}) if PROXY: p = urllib2.ProxyHandler({PROXY[1]: PROXY[0]}) o = urllib2.build_opener(p, urllib2.HTTPHandler) urllib2.install_opener(o) connection = urllib2.urlopen(request) except urllib2.HTTPError, e: if e.code == 401: raise HTTP401Authentication if e.code == 403: raise HTTP403Forbidden if e.code == 404: raise HTTP404NotFound raise HTTPError except urllib2.URLError, e: if e.reason[0] == 36: raise URLTimeout raise URLError return connection #print open("http://nodebox.net") #print open("http:/nodebox.net") #print open("http://ndoebox.net") #print open("http://nodebox.net/doink") #print open("url.py").info() #print open("boink.py").info() #print open("file://url.py").info() ### URL VALIDATION ################################################################################### def is_url(url, wait=10): """ Returns False when no connection can be opened to the url. """ try: connection = open(url, wait) except: return False return True def not_found(url, wait=10): """ Returns True when the url generates a "404 Not Found" error. """ try: connection = open(url, wait) except HTTP404NotFound: return True except: return False return False #url = "http://ndoebox.net" #print is_url(url) #print not_found("http://nodebox.net/nonexistent.html") #print not_found("http://nodebox.net/") ### URL MIMETYPE ################################################################################### def is_type(url, types=[], wait=10): """ Determine the MIME-type of the document behind the url. MIME is more reliable than simply checking the document extension. Returns True when the MIME-type starts with anything in the list of types. """ # Types can also be a single string for convenience. if isinstance(types, str): types = [types] try: connection = open(url, wait) except: return False type = connection.info()["Content-Type"] for t in types: if type.startswith(t): return True return False def is_webpage(url, wait=10): return is_type(url, "text/html", wait) is_page = is_webpage def is_stylesheet(url, wait=10): return is_type(url, "text/css", wait) def is_plaintext(url, wait=10): return is_type(url, "text/plain", wait) def is_pdf(url, wait=10): return is_type(url, "application/pdf", wait) def is_newsfeed(url, wait=10): return is_type(url, ["application/rss+xml", "application/atom+xml"], wait) def is_image(url, wait=10): return is_type(url, ["image/gif", "image/jpeg", "image/x-png"], wait) def is_audio(url, wait=10): return is_type(url, ["audio/mpeg", "audio/x-aiff", "audio/x-wav"], wait) def is_video(url, wait=10): return is_type(url, ["video/mpeg", "video/quicktime"], wait) def is_archive(url, wait=10): return is_type(url, ["application/x-stuffit", "application/x-tar", "application/zip"], wait) #print is_webpage("http://nodebox.net") #print is_archive("http://nodebox.net/code/data/media/coreimage.zip") ### URLACCUMULATOR ################################################################################### urlaccumulator_throttle = {} class URLAccumulator: def __init__(self, url, wait=60, asynchronous=False, cache=None, type=".html", throttle=0): """ Creates a threaded connection to a url and reads data. URLAccumulator can run asynchronously which is useful for animations. The done property is set to True when downloading is complete. The error attribute contains a URLError exception when no data is found. URLAccumulator data can be cached. Downloads that resulted in an error will write an empty file to the cache, the data property will be an empty string but no error is logged when the data is read from the cache in later calls. URLAccumulator can be throttled. This ensures only a certain amount of requests to a domain will happen in a given period of time. URLAccumulator data is loaded. It has a load() method that is called once when done. """ self.url = url self.data = None self.redirect = None self.error = None if cache != None: self.cached = True self._cache = Cache(cache, type) else: self.cached = False self._cache = None self._domain = URLParser(self.url).domain self._throttle = throttle global urlaccumulator_throttle if not self._domain in urlaccumulator_throttle: urlaccumulator_throttle[self._domain] = time.time() - self._throttle self._start = time.time() self._wait = wait self._busy = True self._loaded = False # Synchronous downloads wait until completed, # otherwise check the done property. thread.start_new_thread(self._retrieve, (self.url,)) if not asynchronous: while not self._done(): time.sleep(0.1) def _queued(self): # Throttles live requests: # waits until the current time is greater than # the time of the last request plus the throttle amount. global urlaccumulator_throttle if self.cached and self._cache.exists(str(self.url)): return False elif time.time() < urlaccumulator_throttle[self._domain] + self._throttle: return True else: urlaccumulator_throttle[self._domain] = time.time() return False def _retrieve(self, url): # When the url data is stored in cache, load that. # Otherwise, retrieve it from the web. if self.cached and self._cache.exists(str(url)): self.data = self._cache.read(str(url)) else: try: connection = open(url) self.data = connection.read() self.redirect = connection.geturl() if self.redirect == str(url): self.redirect = None except Exception, e: self.data = u"" self.error = e self._busy = False def _done(self): # Will continue downloading asynchronously. # 1) When the time limit is exceeded, logs a Timeout error. # 2) Once uncached data is ready, stores it in cache. # 3) Loads the data. # 4) Issues a warning when an error occured. if (not self._busy or time.time() >= self._start + self._wait) \ and not self._queued(): # 1 if self.data == None and \ self.error == None: self.data = u"" self.error = URLTimeout() self.load(self.data) self._busy = False if self.cached and not self._cache.exists(str(self.url)) \ and self.data != None and self.data != "": # 2 self._cache.write(str(self.url), str(self.data)) if not self._loaded and self.error == None: # 3 self.load(self.data) self._loaded = True if self.error != None: # 4 warn(str(self.error)+" in "+str(self.__class__)+" for "+str(self.url), Warning) return True else: return False done = property(_done) def load(self, data): """ Override this method in subclasses to process downloaded data. """ pass def retrieve(url, wait=60, asynchronous=False, cache=None, type=".html"): ua = URLAccumulator(url, wait, asynchronous, cache, type) return ua #r = retrieve("http://nodebox.net") #print r.data #print r.redirect #url = create("http://api.search.yahoo.com/ContentAnalysisService/V1/termExtraction", method="post") #url.query["appid"] = "YahooDemo" #url.query["context"] = "Italian sculptors and painters of the renaissance favored the Virgin Mary for inspiration" #url.query["query"] = "madonna" #r = retrieve(url) #print r.data #r = retrieve("http://nodebox.net", asynchronous=True) #while not r.done: # print "wait...", # time.sleep(0.1) #print r.redirect # XXX - should or should we not do quote_plus() somewhere in here?