Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1

2

3

4

5

6

7

8

9

10

11

12

13

14

15

16

17

18

19

20

21

22

23

24

25

26

27

28

29

30

31

32

33

34

35

36

37

38

39

40

41

42

43

44

45

46

47

48

49

50

51

52

53

54

55

56

57

58

59

60

61

62

63

64

65

66

67

68

69

70

71

72

""" 

Helper functions for serializing (and deserializing) requests. 

""" 

 

from scrapy.http import Request 

 

def request_to_dict(request, spider=None): 

    """Convert Request object to a dict. 

 

    If a spider is given, it will try to find out the name of the spider method 

    used in the callback and store that as the callback. 

    """ 

    cb = request.callback 

    if callable(cb): 

        cb = _find_method(spider, cb) 

    eb = request.errback 

    if callable(eb): 

        eb = _find_method(spider, eb) 

    d = { 

        'url': request.url.decode('ascii'), # urls should be safe (safe_string_url) 

        'callback': cb, 

        'errback': eb, 

        'method': request.method, 

        'headers': dict(request.headers), 

        'body': request.body, 

        'cookies': request.cookies, 

        'meta': request.meta, 

        '_encoding': request._encoding, 

        'priority': request.priority, 

        'dont_filter': request.dont_filter, 

    } 

    return d 

 

 

def request_from_dict(d, spider=None): 

    """Create Request object from a dict. 

 

    If a spider is given, it will try to resolve the callbacks looking at the 

    spider for methods with the same name. 

    """ 

    cb = d['callback'] 

    if cb and spider: 

        cb = _get_method(spider, cb) 

    eb = d['errback'] 

    if eb and spider: 

        eb = _get_method(spider, eb) 

    return Request( 

        url=d['url'].encode('ascii'), 

        callback=cb, 

        errback=eb, 

        method=d['method'], 

        headers=d['headers'], 

        body=d['body'], 

        cookies=d['cookies'], 

        meta=d['meta'], 

        encoding=d['_encoding'], 

        priority=d['priority'], 

        dont_filter=d['dont_filter']) 

 

 

def _find_method(obj, func): 

    if obj and hasattr(func, 'im_self') and func.im_self is obj: 

        return func.im_func.__name__ 

    else: 

        raise ValueError("Function %s is not a method of: %s" % (func, obj)) 

 

def _get_method(obj, name): 

    name = str(name) 

    try: 

        return getattr(obj, name) 

    except AttributeError: 

        raise ValueError("Method %r not found in: %s" % (name, obj))