1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
|
from twisted.internet import defer
from twisted.internet.base import ThreadedResolver
from scrapy.utils.datatypes import LocalCache
# TODO: cache misses
dnscache = LocalCache(10000)
class CachingThreadedResolver(ThreadedResolver):
def __init__(self, reactor, cache_size, timeout):
super(CachingThreadedResolver, self).__init__(reactor)
dnscache.limit = cache_size
self.timeout = timeout
def getHostByName(self, name, timeout=None):
if name in dnscache:
return defer.succeed(dnscache[name])
if not timeout:
timeout = self.timeout
d = super(CachingThreadedResolver, self).getHostByName(name, timeout)
d.addCallback(self._cache_result, name)
return d
def _cache_result(self, result, name):
dnscache[name] = result
return result
|
