Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1

2

3

4

5

6

7

8

9

10

11

12

13

14

15

16

17

18

19

20

21

22

23

24

25

26

27

28

29

30

31

32

33

34

35

36

37

38

39

""" 

Scheduler queues 

""" 

 

import marshal, cPickle as pickle 

 

from scrapy.utils import queue 

 

def _serializable_queue(queue_class, serialize, deserialize): 

 

    class SerializableQueue(queue_class): 

 

        def push(self, obj): 

            s = serialize(obj) 

            super(SerializableQueue, self).push(s) 

 

        def pop(self): 

            s = super(SerializableQueue, self).pop() 

            if s: 

                return deserialize(s) 

 

    return SerializableQueue 

 

def _pickle_serialize(obj): 

    try: 

        return pickle.dumps(obj, protocol=2) 

    except pickle.PicklingError, e: 

        raise ValueError(str(e)) 

 

PickleFifoDiskQueue = _serializable_queue(queue.FifoDiskQueue, \ 

    _pickle_serialize, pickle.loads) 

PickleLifoDiskQueue = _serializable_queue(queue.LifoDiskQueue, \ 

    _pickle_serialize, pickle.loads) 

MarshalFifoDiskQueue = _serializable_queue(queue.FifoDiskQueue, \ 

    marshal.dumps, marshal.loads) 

MarshalLifoDiskQueue = _serializable_queue(queue.LifoDiskQueue, \ 

    marshal.dumps, marshal.loads) 

FifoMemoryQueue = queue.FifoMemoryQueue 

LifoMemoryQueue = queue.LifoMemoryQueue