Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1

2

3

4

5

6

7

8

9

10

11

12

13

14

15

16

17

18

19

20

21

22

23

24

25

26

27

28

29

from scrapy.exceptions import NotConfigured 

from scrapy.utils.request import request_httprepr 

from scrapy.utils.response import response_httprepr 

from scrapy.stats import stats 

from scrapy.conf import settings 

 

class DownloaderStats(object): 

 

    def __init__(self): 

11        if not settings.getbool('DOWNLOADER_STATS'): 

            raise NotConfigured 

 

    def process_request(self, request, spider): 

        stats.inc_value('downloader/request_count', spider=spider) 

        stats.inc_value('downloader/request_method_count/%s' % request.method, spider=spider) 

        reqlen = len(request_httprepr(request)) 

        stats.inc_value('downloader/request_bytes', reqlen, spider=spider) 

 

    def process_response(self, request, response, spider): 

        stats.inc_value('downloader/response_count', spider=spider) 

        stats.inc_value('downloader/response_status_count/%s' % response.status, spider=spider) 

        reslen = len(response_httprepr(response)) 

        stats.inc_value('downloader/response_bytes', reslen, spider=spider) 

        return response 

 

    def process_exception(self, request, exception, spider): 

        ex_class = "%s.%s" % (exception.__class__.__module__, exception.__class__.__name__) 

        stats.inc_value('downloader/exception_count', spider=spider) 

        stats.inc_value('downloader/exception_type_count/%s' % ex_class, spider=spider)