Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
""" This module contains data types used by Scrapy which are not included in the Python Standard Library.
This module must not depend on any module outside the Standard Library. """
""" A subclass of dictionary customized to handle multiple values for the same key.
>>> d = MultiValueDict({'name': ['Adrian', 'Simon'], 'position': ['Developer']}) >>> d['name'] 'Simon' >>> d.getlist('name') ['Adrian', 'Simon'] >>> d.get('lastname', 'nonexistent') 'nonexistent' >>> d.setlist('lastname', ['Holovaty', 'Willison'])
This class exists to solve the irritating problem raised by cgi.parse_qs, which returns a list for every key, even though most Web forms submit single name-value pairs. """
""" Returns the last data value for this key, or [] if it's an empty list; raises KeyError if not found. """ except IndexError: return []
dict.__setitem__(self, key, [value])
return self.__class__(dict.items(self))
if memo is None: memo = {} result = self.__class__() memo[id(self)] = result for key, value in dict.items(self): dict.__setitem__(result, copy.deepcopy(key, memo), copy.deepcopy(value, memo)) return result
"Returns the default value if the requested data doesn't exist" if val == []: return default return val
"Returns an empty list if the requested data doesn't exist" except KeyError: return []
if key not in self: self[key] = default return self[key]
if key not in self: self.setlist(key, default_list) return self.getlist(key)
"Appends an item to the internal list associated with key" self.setlistdefault(key, []) dict.__setitem__(self, key, self.getlist(key) + [value])
""" Returns a list of (key, value) pairs, where value is the last item in the list associated with the key. """ return [(key, self[key]) for key in self.keys()]
"Returns a list of (key, list) pairs." return dict.items(self)
"Returns a list of the last value on every key list." return [self[key] for key in self.keys()]
"Returns a copy of this object." return self.__deepcopy__()
"update() extends rather than replaces existing key lists. Also accepts keyword args." if len(args) > 1: raise TypeError, "update expected at most 1 arguments, got %d" % len(args) if args: other_dict = args[0] if isinstance(other_dict, MultiValueDict): for key, value_list in other_dict.lists(): self.setlistdefault(key, []).extend(value_list) else: try: for key, value in other_dict.items(): self.setlistdefault(key, []).append(value) except TypeError: raise ValueError, "MultiValueDict.update() takes either a MultiValueDict or dictionary" for key, value in kwargs.iteritems(): self.setlistdefault(key, []).append(value)
"""Class to represent a site node (page, image or any other file)"""
self.url = url self.itemnames = [] self.children = [] self.parent = None
self.children.append(node) node.parent = self
s = "%s%s\n" % (' '*level, self.url) if self.itemnames: for n in self.itemnames: s += "%sScraped: %s\n" % (' '*(level+1), n) for node in self.children: s += node.to_string(level+1) return s
"""Method to normalize dictionary key access"""
"""Method to normalize values prior to be setted"""
""" A simple class for creating new "virtual" dictionaries that actually look up values in more than one dictionary, passed in the constructor.
If a key appears in more than one of the given dictionaries, only the first occurrence will be used. """
raise KeyError
return self.__class__(*self.dicts)
try: return self[key] except KeyError: return default
for dict_ in self.dicts: if key in dict_.keys(): return dict_.getlist(key) return []
item_list = [] for dict_ in self.dicts: item_list.extend(dict_.items()) return item_list
return False
"""Returns a copy of this object.""" return self.__copy__()
"""Dictionary with a finite number of keys.
Older items expires first.
"""
while len(self) >= self.limit: self.popitem(last=False) super(LocalCache, self).__setitem__(key, value) |