You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
31 lines
1.1 KiB
31 lines
1.1 KiB
from twisted.internet import defer
|
|
from twisted.internet.base import ThreadedResolver
|
|
|
|
from scrapy.utils.datatypes import LocalCache
|
|
|
|
# TODO: cache misses
|
|
|
|
dnscache = LocalCache(10000)
|
|
|
|
class CachingThreadedResolver(ThreadedResolver):
|
|
def __init__(self, reactor, cache_size, timeout):
|
|
super(CachingThreadedResolver, self).__init__(reactor)
|
|
dnscache.limit = cache_size
|
|
self.timeout = timeout
|
|
|
|
def getHostByName(self, name, timeout=None):
|
|
if name in dnscache:
|
|
return defer.succeed(dnscache[name])
|
|
# in Twisted<=16.6, getHostByName() is always called with
|
|
# a default timeout of 60s (actually passed as (1, 3, 11, 45) tuple),
|
|
# so the input argument above is simply overridden
|
|
# to enforce Scrapy's DNS_TIMEOUT setting's value
|
|
timeout = (self.timeout,)
|
|
d = super(CachingThreadedResolver, self).getHostByName(name, timeout)
|
|
if dnscache.limit:
|
|
d.addCallback(self._cache_result, name)
|
|
return d
|
|
|
|
def _cache_result(self, result, name):
|
|
dnscache[name] = result
|
|
return result
|