Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1

2

3

4

5

6

7

8

9

10

11

12

13

14

15

16

17

18

19

20

21

22

23

24

import socket 

 

from twisted.internet import defer 

from twisted.internet.base import ThreadedResolver 

 

from scrapy.utils.datatypes import LocalCache 

 

# TODO: cache misses 

# TODO: make cache size a setting 

 

dnscache = LocalCache(10000) 

 

class CachingThreadedResolver(ThreadedResolver): 

 

    def getHostByName(self, name, timeout = (1, 3, 11, 45)): 

        if name in dnscache: 

            return defer.succeed(dnscache[name]) 

        d = ThreadedResolver.getHostByName(self, name, timeout) 

        d.addCallback(self._cache_result, name) 

        return d 

 

    def _cache_result(self, result, name): 

        dnscache[name] = result 

        return result