Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1

2

3

4

5

6

7

8

9

10

11

12

13

14

15

16

17

18

19

20

21

22

23

24

25

26

27

28

29

30

31

32

33

34

35

36

37

38

39

40

41

42

43

44

45

46

47

48

49

50

51

52

53

54

55

56

57

58

59

60

61

62

63

64

65

66

67

68

69

70

71

72

73

74

75

76

77

78

79

80

81

82

83

84

85

86

87

88

89

90

91

92

93

94

95

96

97

""" 

Scrapy web services extension 

 

See docs/topics/webservice.rst 

""" 

 

from twisted.web import server, error 

 

from scrapy.xlib.pydispatch import dispatcher 

from scrapy.exceptions import NotConfigured 

from scrapy import log, signals 

from scrapy.utils.jsonrpc import jsonrpc_server_call 

from scrapy.utils.serialize import ScrapyJSONEncoder, ScrapyJSONDecoder 

from scrapy.utils.misc import load_object 

from scrapy.utils.txweb import JsonResource as JsonResource_ 

from scrapy.utils.reactor import listen_tcp 

from scrapy.utils.conf import build_component_list 

 

 

class JsonResource(JsonResource_): 

 

    def __init__(self, crawler, target=None): 

        JsonResource_.__init__(self) 

        self.crawler = crawler 

        self.json_encoder = ScrapyJSONEncoder(crawler=crawler) 

 

class JsonRpcResource(JsonResource): 

 

    def __init__(self, crawler, target=None): 

        JsonResource.__init__(self, crawler, target) 

        self.json_decoder = ScrapyJSONDecoder(crawler=crawler) 

        self.crawler = crawler 

        self._target = target 

 

    def render_GET(self, txrequest): 

        return self.get_target() 

 

    def render_POST(self, txrequest): 

        reqstr = txrequest.content.getvalue() 

        target = self.get_target() 

        return jsonrpc_server_call(target, reqstr, self.json_decoder) 

 

    def getChild(self, name, txrequest): 

        target = self.get_target() 

        try: 

            newtarget = getattr(target, name) 

            return JsonRpcResource(newtarget) 

        except AttributeError: 

            return error.NoResource("No such child resource.") 

 

    def get_target(self): 

        return self._target 

 

 

class RootResource(JsonResource): 

 

    def render_GET(self, txrequest): 

        return {'resources': self.children.keys()} 

 

    def getChild(self, name, txrequest): 

        if name == '': 

            return self 

        return JsonResource.getChild(self, name, txrequest) 

 

 

class WebService(server.Site): 

 

    def __init__(self, crawler): 

70        if not crawler.settings.getbool('WEBSERVICE_ENABLED'): 

            raise NotConfigured 

        self.crawler = crawler 

        logfile = crawler.settings['WEBSERVICE_LOGFILE'] 

        self.portrange = map(int, crawler.settings.getlist('WEBSERVICE_PORT')) 

        self.host = crawler.settings['WEBSERVICE_HOST'] 

        root = RootResource(crawler) 

        reslist = build_component_list(crawler.settings['WEBSERVICE_RESOURCES_BASE'], \ 

            crawler.settings['WEBSERVICE_RESOURCES']) 

        for res_cls in map(load_object, reslist): 

            res = res_cls(crawler) 

            root.putChild(res.ws_name, res) 

        server.Site.__init__(self, root, logPath=logfile) 

        self.noisy = False 

        dispatcher.connect(self.start_listening, signals.engine_started) 

        dispatcher.connect(self.stop_listening, signals.engine_stopped) 

 

    @classmethod 

    def from_crawler(cls, crawler): 

        return cls(crawler) 

 

    def start_listening(self): 

        self.port = listen_tcp(self.portrange, self.host, self) 

        h = self.port.getHost() 

        log.msg("Web service listening on %s:%d" % (h.host, h.port), log.DEBUG) 

 

    def stop_listening(self): 

        self.port.stopListening()