preprocessors.py

import re


class Binarizer(object):
    '''Callable for transforming "truthy/falsey" values to 0/1 respectively.
    '''
    def __init__(self, **kwargs):
        pass

    def __call__(self, value):
        return int(bool(value))


class FlatMap(object):
    '''Callable for transforming a nested structure composted of dictionaries
    and lists into a single list of unique paths. The return value is always
    a list; an input that is a primitive (string or numeric) value is
    transformed into a list containing that value.
    '''
    def __init__(self, **kwargs):
        pass

    def __call__(self, document):
        state = []

        def recur(key, value):
            # Unpacks the `value` object to recursively add keys to the state
            if isinstance(value, dict):
                for child, new_value in value.items():
                    new_key = '.'.join(filter(None, [key, child]))
                    recur(new_key, new_value)
            elif isinstance(value, list):
                for child in value:
                    recur(key, child)
            else:
                state.append(':'.join(filter(None, [key, str(value)])))

        recur('', document)
        return state


class Tokenizer(object):
    '''Transforms a string into a list of "canonical" tokens via two
    operations: (1) normalizing to lower case, and (2) extracting only
    matches of a single regex.
    '''
    def __init__(self, regex='\\S+', **kwargs):
        self._regex = re.compile(regex)

    def __call__(self, document):
        state = []

        def recur(datum):
            if isinstance(datum, str):
                state.extend(self._regex.findall(datum.lower()))
            elif isinstance(datum, list):
                for child in datum:
                    recur(child)

        recur(document)
        return list(filter(None, state))


class PassThrough(object):
    '''Implements the identity transform.'''
    def __init__(self, **kwargs):
        pass

    def __call__(self, document):
        return document