Coverage for scrapy/utils/misc : 95%

"""Helper functions which doesn't fit anywhere else""" 

import re 

import hashlib 

from pkgutil import iter_modules 

from w3lib.html import remove_entities 

from scrapy.utils.python import flatten 

def arg_to_iter(arg): 

    """Convert an argument to an iterable. The argument can be a None, single 

    value, or an iterable. 

    Exception: if arg is a dict, [arg] will be returned 

    """ 

    if arg is None: 

        return [] 

    elif not isinstance(arg, dict) and hasattr(arg, '__iter__'): 

        return arg 

    else: 

        return [arg] 

def load_object(path): 

    """Load an object given its absolute object path, and return it. 

    object can be a class, function, variable o instance. 

    path ie: 'scrapy.contrib.downloadermiddelware.redirect.RedirectMiddleware' 

    """ 

    try: 

        dot = path.rindex('.') 

    except ValueError: 

        raise ValueError, "Error loading object '%s': not a full path" % path 

    module, name = path[:dot], path[dot+1:] 

    try: 

        mod = __import__(module, {}, {}, ['']) 

    except ImportError, e: 

        raise ImportError, "Error loading object '%s': %s" % (path, e) 

    try: 

        obj = getattr(mod, name) 

    except AttributeError: 

        raise NameError, "Module '%s' doesn't define any object named '%s'" % (module, name) 

    return obj 

def walk_modules(path, load=False): 

    """Loads a module and all its submodules from a the given module path and 

    returns them. If *any* module throws an exception while importing, that 

    exception is thrown back. 

    For example: walk_modules('scrapy.utils') 

    """ 

    mods = [] 

    mod = __import__(path, {}, {}, ['']) 

    mods.append(mod) 

    if hasattr(mod, '__path__'): 

        for _, subpath, ispkg in iter_modules(mod.__path__): 

            fullpath = path + '.' + subpath 

            if ispkg: 

                mods += walk_modules(fullpath) 

            else: 

                submod = __import__(fullpath, {}, {}, ['']) 

                mods.append(submod) 

    return mods 

def extract_regex(regex, text, encoding='utf-8'): 

    """Extract a list of unicode strings from the given text/encoding using the following policies: 

    * if the regex contains a named group called "extract" that will be returned 

    * if the regex contains multiple numbered groups, all those will be returned (flattened) 

    * if the regex doesn't contain any group the entire regex matching is returned 

    """ 

    if isinstance(regex, basestring): 

        regex = re.compile(regex, re.UNICODE) 

    try: 

        strings = [regex.search(text).group('extract')]   # named group 

    except: 

        strings = regex.findall(text)    # full regex or numbered groups 

    strings = flatten(strings) 

89    if isinstance(text, unicode): 

        return [remove_entities(s, keep=['lt', 'amp']) for s in strings] 

    else: 

        return [remove_entities(unicode(s, encoding), keep=['lt', 'amp']) for s in strings] 

def md5sum(file): 

    """Calculate the md5 checksum of a file-like object without reading its 

    whole content in memory. 

    >>> from StringIO import StringIO 

    >>> md5sum(StringIO('file content to hash')) 

    '784406af91dd5a54fbb9c84c2236595a' 

    """ 

    m = hashlib.md5() 

    while 1: 

        d = file.read(8096) 

        if not d: 

            break 

        m.update(d) 

    return m.hexdigest()