"""
Preprocess
----------

"""
import numpy as np
import bottleneck as bn
import scipy.sparse as sp
from sklearn.impute import SimpleImputer

import Orange.data
from Orange.data.filter import HasClass
from Orange.statistics import distribution
from Orange.util import Reprable, Enum, deprecated
from . import impute, discretize, transformation

__all__ = ["Continuize", "Discretize", "Impute", "RemoveNaNRows",
           "SklImpute", "Normalize", "Randomize", "Preprocess",
           "RemoveConstant", "RemoveNaNClasses", "RemoveNaNColumns",
           "ProjectPCA", "ProjectCUR", "Scale", "RemoveSparse",
           "AdaptiveNormalize", "PreprocessorList"]


class Preprocess(Reprable):
    """
    A generic preprocessor base class.

    Methods
    -------
    __call__(data: Table) -> Table
        Return preprocessed data.
    """
    def __call__(self, data):
        raise NotImplementedError("Subclasses need to implement __call__")


class Continuize(Preprocess):
    MultinomialTreatment = Enum(
        "Continuize",
        ("Indicators", "FirstAsBase", "FrequentAsBase", "Remove",
         "RemoveMultinomial", "ReportError", "AsOrdinal", "AsNormalizedOrdinal",
         "Leave"),
        qualname="Continuize.MultinomialTreatment")
    (Indicators, FirstAsBase, FrequentAsBase, Remove, RemoveMultinomial,
     ReportError, AsOrdinal, AsNormalizedOrdinal, Leave) = MultinomialTreatment

    def __init__(self, zero_based=True,
                 multinomial_treatment=Indicators):
        self.zero_based = zero_based
        self.multinomial_treatment = multinomial_treatment

    def __call__(self, data):
        from . import continuize

        continuizer = continuize.DomainContinuizer(
            zero_based=self.zero_based,
            multinomial_treatment=self.multinomial_treatment)
        domain = continuizer(data)
        return data.transform(domain)


class Discretize(Preprocess):
    """
    Construct a discretizer, a preprocessor for discretization of
    continuous features.

    Parameters
    ----------
    method : discretization method (default: Orange.preprocess.discretize.Discretization)

    remove_const : bool (default=True)
        Determines whether the features with constant values are removed
        during discretization.
    """

    def __init__(self, method=None, remove_const=True,
                 discretize_classes=False, discretize_metas=False):
        self.method = method
        self.remove_const = remove_const
        self.discretize_classes = discretize_classes
        self.discretize_metas = discretize_metas

    def __call__(self, data):
        """
        Compute and apply discretization of the given data. Returns a new
        data table.

        Parameters
        ----------
        data : Orange.data.Table
            A data table to be discretized.
        """

        def transform(var):
            if var.is_continuous:
                new_var = method(data, var)
                if new_var is not None and \
                        (len(new_var.values) >= 2 or not self.remove_const):
                    return new_var
                else:
                    return None
            else:
                return var

        def discretized(vars_, do_discretize):
            if do_discretize:
                vars_ = (transform(var) for var in vars_)
                vars_ = [var for var in vars_ if var is not None]
            return vars_

        method = self.method or discretize.EqualFreq()
        domain = Orange.data.Domain(
            discretized(data.domain.attributes, True),
            discretized(data.domain.class_vars, self.discretize_classes),
            discretized(data.domain.metas, self.discretize_metas))
        return data.transform(domain)


class Impute(Preprocess):
    """
    Construct a imputer, a preprocessor for imputation of missing values in
    the data table.

    Parameters
    ----------
    method : imputation method (default: Orange.preprocess.impute.Average())
    """

    def __init__(self, method=Orange.preprocess.impute.Average()):
        self.method = method

    def __call__(self, data):
        """
        Apply an imputation method to the given dataset. Returns a new
        data table with missing values replaced by their imputations.

        Parameters
        ----------
        data : Orange.data.Table
            An input data table.
        """

        method = self.method or impute.Average()
        newattrs = [method(data, var) for var in data.domain.attributes]
        domain = Orange.data.Domain(
            newattrs, data.domain.class_vars, data.domain.metas)
        return data.transform(domain)


class SklImpute(Preprocess):
    __wraps__ = SimpleImputer

    def __init__(self, strategy='mean'):
        self.strategy = strategy

    def __call__(self, data):
        from Orange.data.sql.table import SqlTable
        if isinstance(data, SqlTable):
            return Impute()(data)
        imputer = SimpleImputer(strategy=self.strategy)
        imputer.fit(data.X)
        # Create new variables with appropriate `compute_value`, but
        # drop the ones which do not have valid `imputer.statistics_`
        # (i.e. all NaN columns). `sklearn.preprocessing.Imputer` already
        # drops them from the transformed X.
        features = [var.copy(compute_value=impute.ReplaceUnknowns(var, value))
                    for var, value in zip(data.domain.attributes,
                                          imputer.statistics_)
                    if not np.isnan(value)]
        domain = Orange.data.Domain(features, data.domain.class_vars,
                                    data.domain.metas)
        new_data = data.transform(domain)
        return new_data


class RemoveConstant(Preprocess):
    """
    Construct a preprocessor that removes features with constant values
    from the dataset.
    """

    def __call__(self, data):
        """
        Remove columns with constant values from the dataset and return
        the resulting data table.

        Parameters
        ----------
        data : an input dataset
        """

        oks = np.logical_and(~bn.allnan(data.X, axis=0),
                             bn.nanmin(data.X, axis=0) != bn.nanmax(data.X, axis=0))
        atts = [data.domain.attributes[i] for i, ok in enumerate(oks) if ok]
        domain = Orange.data.Domain(atts, data.domain.class_vars,
                                    data.domain.metas)
        return data.transform(domain)


class RemoveNaNRows(Preprocess):
    _reprable_module = True

    def __call__(self, data):
        mask = np.isnan(data.X)
        mask = np.any(mask, axis=1)
        return data[~mask]


class RemoveNaNColumns(Preprocess):
    """
    Remove features from the data domain if they contain
    `threshold` or more unknown values.

    `threshold` can be an integer or a float in the range (0, 1) representing
    the fraction of the data size. When not provided, columns with only missing
    values are removed (default).
    """
    def __init__(self, threshold=None):
        self.threshold = threshold

    def __call__(self, data, threshold=None):
        # missing entries in sparse data are treated as zeros so we skip removing NaNs
        if sp.issparse(data.X):
            return data

        if threshold is None:
            threshold = data.X.shape[0] if self.threshold is None else \
                        self.threshold
        if isinstance(threshold, float):
            threshold = threshold * data.X.shape[0]
        nans = np.sum(np.isnan(data.X), axis=0)
        att = [a for a, n in zip(data.domain.attributes, nans) if n < threshold]
        domain = Orange.data.Domain(att, data.domain.class_vars,
                                    data.domain.metas)
        return data.transform(domain)


@deprecated("Orange.data.filter.HasClas")
class RemoveNaNClasses(Preprocess):
    """
    Construct preprocessor that removes examples with missing class
    from the dataset.
    """

    def __call__(self, data):
        """
        Remove rows that contain NaN in any class variable from the dataset
        and return the resulting data table.

        Parameters
        ----------
        data : an input dataset

        Returns
        -------
        data : dataset without rows with missing classes
        """
        return HasClass()(data)


class Normalize(Preprocess):
    """
    Construct a preprocessor for normalization of features.
    Given a data table, preprocessor returns a new table in
    which the continuous attributes are normalized.

    Parameters
    ----------
    zero_based : bool (default=True)
        Only used when `norm_type=NormalizeBySpan`.

        Determines the value used as the “low” value of the variable.
        It determines the interval for normalized continuous variables
        (either [-1, 1] or [0, 1]).

    norm_type : NormTypes (default: Normalize.NormalizeBySD)
        Normalization type. If Normalize.NormalizeBySD, the values are
        replaced with standardized values by subtracting the average
        value and dividing by the standard deviation.
        Attribute zero_based has no effect on this standardization.

        If Normalize.NormalizeBySpan, the values are replaced with
        normalized values by subtracting min value of the data and
        dividing by span (max - min).

    transform_class : bool (default=False)
        If True the class is normalized as well.

    center : bool (default=True)
        Only used when `norm_type=NormalizeBySD`.

        Whether or not to center the data so it has mean zero.

    normalize_datetime : bool (default=False)


    Examples
    --------
    >>> from Orange.data import Table
    >>> from Orange.preprocess import Normalize
    >>> data = Table("iris")
    >>> normalizer = Normalize(norm_type=Normalize.NormalizeBySpan)
    >>> normalized_data = normalizer(data)
    """
    Type = Enum("Normalize", ("NormalizeBySpan", "NormalizeBySD"),
                qualname="Normalize.Type")
    NormalizeBySpan, NormalizeBySD = Type

    def __init__(self,
                 zero_based=True,
                 norm_type=NormalizeBySD,
                 transform_class=False,
                 center=True,
                 normalize_datetime=False):
        self.zero_based = zero_based
        self.norm_type = norm_type
        self.transform_class = transform_class
        self.center = center
        self.normalize_datetime = normalize_datetime

    def __call__(self, data):
        """
        Compute and apply normalization of the given data. Returns a new
        data table.

        Parameters
        ----------
        data : Orange.data.Table
            A data table to be normalized.

        Returns
        -------
        data : Orange.data.Table
            Normalized data table.
        """
        from . import normalize

        if all(a.attributes.get('skip-normalization', False)
               for a in data.domain.attributes if a.is_continuous):
            # Skip normalization for datasets where all features are marked as already normalized.
            # Required for SVMs (with normalizer as their default preprocessor) on sparse data to
            # retain sparse structure. Normalizing sparse data would otherwise result in a dense
            # matrix, which requires too much memory. For example, this is used for Bag of Words
            # models where normalization is not really needed.
            return data

        normalizer = normalize.Normalizer(
            zero_based=self.zero_based,
            norm_type=self.norm_type,
            transform_class=self.transform_class,
            center=self.center,
            normalize_datetime=self.normalize_datetime
        )
        return normalizer(data)


class Randomize(Preprocess):
    """
    Construct a preprocessor for randomization of classes,
    attributes and/or metas.
    Given a data table, preprocessor returns a new table in
    which the data is shuffled.

    Parameters
    ----------

    rand_type : RandTypes (default: Randomize.RandomizeClasses)
        Randomization type. If Randomize.RandomizeClasses, classes
        are shuffled.
        If Randomize.RandomizeAttributes, attributes are shuffled.
        If Randomize.RandomizeMetas, metas are shuffled.

    rand_seed : int (optional)
        Random seed

    Examples
    --------
    >>> from Orange.data import Table
    >>> from Orange.preprocess import Randomize
    >>> data = Table("iris")
    >>> randomizer = Randomize(Randomize.RandomizeClasses)
    >>> randomized_data = randomizer(data)
    """
    Type = Enum("Randomize",
                dict(RandomizeClasses=1,
                     RandomizeAttributes=2,
                     RandomizeMetas=4),
                type=int,
                qualname="Randomize.Type")
    RandomizeClasses, RandomizeAttributes, RandomizeMetas = Type

    def __init__(self, rand_type=RandomizeClasses, rand_seed=None):
        self.rand_type = rand_type
        self.rand_seed = rand_seed

    def __call__(self, data):
        """
        Apply randomization of the given data. Returns a new
        data table.

        Parameters
        ----------
        data : Orange.data.Table
            A data table to be randomized.

        Returns
        -------
        data : Orange.data.Table
            Randomized data table.
        """
        new_data = data.copy()
        rstate = np.random.RandomState(self.rand_seed)
        # ensure the same seed is not used to shuffle X and Y at the same time
        r1, r2, r3 = rstate.randint(0, 2 ** 32 - 1, size=3, dtype=np.int64)
        with new_data.unlocked():
            if self.rand_type & Randomize.RandomizeClasses:
                new_data.Y = self.randomize(new_data.Y, r1)
            if self.rand_type & Randomize.RandomizeAttributes:
                new_data.X = self.randomize(new_data.X, r2)
            if self.rand_type & Randomize.RandomizeMetas:
                new_data.metas = self.randomize(new_data.metas, r3)
        return new_data

    @staticmethod
    def randomize(table, rand_state=None):
        rstate = np.random.RandomState(rand_state)
        if sp.issparse(table):
            table = table.tocsc()  # type: sp.spmatrix
            for i in range(table.shape[1]):
                permutation = rstate.permutation(table.shape[0])
                col_indices = \
                    table.indices[table.indptr[i]: table.indptr[i + 1]]
                col_indices[:] = permutation[col_indices]
        elif len(table.shape) > 1:
            for i in range(table.shape[1]):
                rstate.shuffle(table[:, i])
        else:
            rstate.shuffle(table)
        return table


class ProjectPCA(Preprocess):

    def __init__(self, n_components=None):
        self.n_components = n_components

    def __call__(self, data):
        pca = Orange.projection.PCA(n_components=self.n_components)(data)
        return pca(data)


class ProjectCUR(Preprocess):

    def __init__(self, rank=3, max_error=1):
        self.rank = rank
        self.max_error = max_error

    def __call__(self, data):
        rank = min(self.rank, min(data.X.shape)-1)
        cur = Orange.projection.CUR(
            rank=rank, max_error=self.max_error,
            compute_U=False,
        )(data)
        return cur(data)


class Scale(Preprocess):
    """
    Scale data preprocessor.  Scales data so that its distribution remains
    the same but its location on the axis changes.
    """
    class _MethodEnum(Enum):
        def __call__(self, *args, **kwargs):
            return getattr(Scale, '_' + self.name)(*args, **kwargs)

    CenteringType = _MethodEnum("Scale", ("NoCentering", "Mean", "Median"),
                                qualname="Scale.CenteringType")
    ScalingType = _MethodEnum("Scale", ("NoScaling", "Std", "Span"),
                              qualname="Scale.ScalingType")
    NoCentering, Mean, Median = CenteringType
    NoScaling, Std, Span = ScalingType

    @staticmethod
    def _Mean(dist):
        values, counts = np.array(dist)
        return np.average(values, weights=counts)

    @staticmethod
    def _Median(dist):
        values, counts = np.array(dist)
        cumdist = np.cumsum(counts)
        if cumdist[-1] > 0:
            cumdist /= cumdist[-1]
        return np.interp(.5, cumdist, values)

    @staticmethod
    def _Std(dist):
        values, counts = np.array(dist)
        mean = np.average(values, weights=counts)
        diff = values - mean
        return np.sqrt(np.average(diff ** 2, weights=counts))

    @staticmethod
    def _Span(dist):
        values = np.array(dist[0])
        return np.max(values) - np.min(values)

    def __init__(self, center=Mean, scale=Std):
        self.center = center
        self.scale = scale

    def __call__(self, data):
        if self.center is None and self.scale is None:
            return data

        def transform(var):
            dist = distribution.get_distribution(data, var)
            if self.center != self.NoCentering:
                c = self.center(dist)
                dist[0, :] -= c
            else:
                c = 0

            if self.scale != self.NoScaling:
                s = self.scale(dist)
                if s < 1e-15:
                    s = 1
            else:
                s = 1
            factor = 1 / s
            transformed_var = var.copy(
                compute_value=transformation.Normalizer(var, c, factor))
            return transformed_var

        newvars = []
        for var in data.domain.attributes:
            if var.is_continuous:
                newvars.append(transform(var))
            else:
                newvars.append(var)
        domain = Orange.data.Domain(newvars, data.domain.class_vars,
                                    data.domain.metas)
        return data.transform(domain)


class PreprocessorList(Preprocess):
    """
    Store a list of preprocessors and on call apply them to the dataset.

    Parameters
    ----------
    preprocessors : list
        A list of preprocessors.
    """

    def __init__(self, preprocessors=()):
        self.preprocessors = list(preprocessors)

    def __call__(self, data):
        """
        Applies a list of preprocessors to the dataset.

        Parameters
        ----------
        data : an input data table
        """

        for pp in self.preprocessors:
            data = pp(data)
        return data

class RemoveSparse(Preprocess):
    """
    Filter out the features with too many (>threshold) zeros or missing values. Threshold is user defined.

    Parameters
    ----------
    threshold: int or float
        if >= 1, the argument represents the allowed number of 0s or NaNs;
        if below 1, it represents the allowed proportion of 0s or NaNs
    filter0: bool
        if True (default), preprocessor counts 0s, otherwise NaNs
    """
    def __init__(self, threshold=0.05, filter0=True):
        self.filter0 = filter0
        self.threshold = threshold

    def __call__(self, data):
        threshold = self.threshold
        if self.threshold < 1:
            threshold *= data.X.shape[0]

        if self.filter0:
            if sp.issparse(data.X):
                data_csc = sp.csc_matrix(data.X)
                h, w = data_csc.shape
                sparseness = [h - data_csc[:, i].count_nonzero()
                              for i in range(w)]
            else:
                sparseness = data.X.shape[0] - np.count_nonzero(data.X, axis=0)
        else:  # filter by nans
            if sp.issparse(data.X):
                data_csc = sp.csc_matrix(data.X)
                sparseness = [np.sum(np.isnan(data.X[:, i].data))
                              for i in range(data_csc.shape[1])]
            else:
                sparseness = np.sum(np.isnan(data.X), axis=0)
        att = [a for a, s in zip(data.domain.attributes, sparseness)
               if s <= threshold]
        domain = Orange.data.Domain(att, data.domain.class_vars,
                                    data.domain.metas)
        return data.transform(domain)


class AdaptiveNormalize(Preprocess):
    """
    Construct a preprocessors that normalizes or merely scales the data.
    If the input is sparse, data is only scaled, to avoid turning it to
    dense. Parameters are diveded to those passed to Normalize or Scale
    class. Scaling takes only scale parameter.
     If the user wants to have more options with scaling,
    they should use the preprocessing widget.
    For more details, check Scale and Normalize widget.

    Parameters
    ----------

    zero_based : bool (default=True)
        passed to Normalize

    norm_type : NormTypes (default: Normalize.NormalizeBySD)
        passed to Normalize

    transform_class : bool (default=False)
        passed to Normalize

    center : bool(default=True)
        passed to Normalize

    normalize_datetime : bool (default=False)
        passed to Normalize

    scale : ScaleTypes (default: Scale.Span)
        passed to Scale
    """

    def __init__(self,
                 zero_based=True,
                 norm_type=Normalize.NormalizeBySD,
                 transform_class=False,
                 normalize_datetime=False,
                 center=True,
                 scale=Scale.Span):
        self.normalize_pps = Normalize(zero_based,
                                       norm_type,
                                       transform_class,
                                       center,
                                       normalize_datetime)
        self.scale_pps = Scale(center=Scale.NoCentering, scale=scale)

    def __call__(self, data):
        if sp.issparse(data.X):
            return self.scale_pps(data)
        return self.normalize_pps(data)