discretizer update & utils create

src/tabular/feature_engineering/new_featureEngeneer/discretizer.py

@@ -1,55 +1,176 @@
 # coding = 'utf-8'
-import numpy as np
-from sklearn.utils import check_array
+import pandas as pd
+import warnings
+
 from sklearn.preprocessing import KBinsDiscretizer
+from .utils import get_continue_feature
+
+class dis_configure:
+    """
+    The config object of discretizer. It saves the parameters of discretizer and check their validity.
+
+    Parameters
+    ----------
+    method : {'uniform', 'quantile', 'kmeans'}, (default='quantile')
+        uniform
+            All bins in each feature have identical widths.
+        quantile
+            All bins in each feature have the same number of points.
+        kmeans
+            Values in each bin have the same nearest center of a 1D k-means
+            cluster.
+
+    n_bins : int, default=5
+    index_col : str, default='id'
+        the col of df_list's DataFrame index col
+    """
+
+    method = None
+    n_bins = None
+    index_col = None
+
+    def _check(self):
+        if self.method is None:
+            self.method = "quantile"
+        elif self.method not in ['uniform', 'quantile', 'kmeans']:
+            raise ValueError(
+                "the method value {} is Invalid! It must be in ['uniform', 'quantile', 'kmeans'], "
+                "default is 'quantile'".format(
+                    str(self.method)))
+
+        if self.n_bins is None:
+            self.n_bins = 5
+        elif not isinstance(self.n_bins, int):
+            raise ValueError(
+                "the n_bins value {} is Invalid! It must be Int value "
+                "default is 5".format(
+                    str(self.n_bins)))
+
+        if self.index_col is None:
+            self.index_col = "id"
+        elif not isinstance(self.index_col, str):
+            raise ValueError(
+                "the index_col value {} is Invalid! It must be str value "
+                "default is 'id'".format(
+                    str(self.index_col)))
+
+    def __init__(self, method, n_bins, index_col):
+        self.method = method
+        self.n_bins = n_bins
+        self.index_col = index_col
+        self._check()
 
 
-def discretizer(df_list, names, method_list):
+def check_index_col(data, config):
     """
+    check the index column's values are unique.
+
     Parameters
     ----------
-    df_list: pd.DataFrame type, the dataframe need to split.
-    names: list of column names
-    method_list: a dictionary contains the methods as key, and parameters as values.
-    key must in ['isometric','quantile','kmeans']
-    Like {"isometric":[n_bins]},{"quantile":[n_bins]},{"kmeans":[n_bins]}
+    data : pd.Dataframe
+    config : object,
+        the config parameter object.
+
+    Returns
+    ----------
+    data : pd.Dataframe,
+        origin data
+    """
+
+    index_col_data = data[config.index_col]
+    if index_col_data.shape[1] == index_col_data.drop_duplicates().shape[1]:
+        return data
+    else:
+        raise ValueError("the index column '{}' values must be unique".format(config.index_col))
+
+
+def retrun_df_list(df_list,data,config):
+    """
+
+    Parameters
+    ----------
+    df_list :object
+        a collection of one or more pd.DataFrame. they must have one column named 'id' for indexing.
+    data : pd.DataFrame
+        the DataFrame after discrete
+    config : Object
+        the object of parameters
 
     Returns
     -------
-    discretizers : list type. the discretizers of all column
-    data: the np.array after trans
+    df_list_t :
+        a collection of one or more pd.DataFrame, they are transformed.
     """
-    if names is None:
-        data = check_array(df_list)
+    df_list_t = list()
+    if isinstance(df_list, tuple) or isinstance(df_list, list):
+        for i in range(len(df_list)):
+            df_list_t.append(df_list[i][[config.index_col]].merge(data,on=config.index_col))
+
+    elif isinstance(df_list, pd.DataFrame):
+       df_list_t.append(df_list[[config.index_col]].merge(data,on=config.index_col))
     else:
-        data = check_array(df_list[names])
+        raise ValueError("paramter df_list must be the collection of one or more pd.DataFrame")
 
-    if len(method_list.keys) > 1:
-        raise ValueError("method_list only can has 1 key")
+    return df_list_t
 
-    method = list(method_list.keys)[0]
-    if method not in ("isometric", "quantile", "kmeans"):
-        raise ValueError("`method` must be 'isometric','quantile' or 'kmeans'")
+def concat_df_list(df_list, config):
+    """
+    concat the df_list as one dataframe
+    Parameters
+    ----------
+    df_list : pd.DataFrame or collection of pd.DataFrame
+        the origin collection of pd.DataFrame
+    config : object
+        the object of parameters
 
-    discretizers = []
-    if method == "isometric":
-        for column in range(np.shape(data)[0]):
-            discretizer = KBinsDiscretizer(n_bins=method_list[method], encode="ordinal", strategy="uniform")
-            fit_encoder(column, data, discretizer, discretizers)
+    Returns
+    -------
+    df
 
-    elif method == "quantile":
-        for column in range(np.shape(data)[0]):
-            discretizer = KBinsDiscretizer(n_bins=method_list[method], encode="ordinal", strategy="quantile")
-            fit_encoder(column, data, discretizer, discretizers)
+    """
+    if isinstance(df_list, tuple) or isinstance(df_list, list):
+        data = pd.concat(df_list, axis=1)
+    elif isinstance(df_list, pd.DataFrame):
+        data = df_list
     else:
-        for column in range(np.shape(data)[0]):
-            discretizer = KBinsDiscretizer(n_bins=method_list[method], encode="ordinal", strategy="kmeans")
-            fit_encoder(column, data, discretizer, discretizers)
+        raise ValueError("paramter df_list must be the collection of one or more pd.DataFrame")
+
+    df = check_index_col(data, config)
+    return df
+
+
+def discretizer(df_list, names, config):
+    """
+    concat the df_list as one pd.DataFrame then using sklean.KBinsDiscretizer depart it,
+
+    Parameters
+    ----------
+    df_list : object
+        a collection of one or more pd.DataFrame. they must have one column named 'id' for indexing.
+    names : list
+        a list of the continuous variable column's name. If it's none,checking if all columns are continuous and
+        discrete the continuous variable columns.
+
+    config : object
+        the object of parameters
+
+    Returns
+    ----------
+    df_list_t : object
+        the df_list after trans ,still is the collection of pd.DataFrame
+    """
+    data = concat_df_list(df_list, config)
+
+    if names is None:
+        warnings.warn("The parameter names is None, will check th")
+        names, _ = get_continue_feature(data)
+
+    for name in names:
+        kbdis = KBinsDiscretizer(n_bins=config.n_bins,encode="ordinal",strategy=config.method)
+        kbdis.fit(data[name])
+        data.loc[:,name+"_discred"]=kbdis.transform(data[name])
+
+    return retrun_df_list(df_list,data,config)
 
-    return discretizers, data
 
 
-def fit_encoder(column, data, discretizer, discretizers):
-    discretizer.fit(data[:, column])
-    data[:, column] = discretizer.transform(data[:, column])
-    discretizers.append(discretizer)

src/tabular/feature_engineering/new_featureEngeneer/utils.py

@@ -0,0 +1,16 @@
+# encoding:utf-8
+
+import pandas as pd
+
+def get_continue_feature(data):
+    continus_features, discrete_features = [], []
+    for col in data.columns:
+        if data[col].dtype != object:
+            try:
+                pd.qcut(data[col], 4)
+                continus_features.append(col)
+            except ValueError:
+                discrete_features.append(col)
+    print('Continus features are: ', continus_features)
+    print('Discrete features are: ', discrete_features)
+    return continus_features, discrete_features