From 383e4b34f5f11e23fb934152106362616bd53d77 Mon Sep 17 00:00:00 2001
From: sugangnb <sugangsky@gmail.com>
Date: Wed, 8 Jan 2020 10:56:22 +0800
Subject: [PATCH 1/4] =?UTF-8?q?=E4=BB=A3=E7=A0=81=E6=9B=B4=E6=96=B0?=
 =?UTF-8?q?=E6=B5=8B=E8=AF=95?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 .../new_featureEngeneer/discretizer.py        | 49 ++-----------------
 1 file changed, 3 insertions(+), 46 deletions(-)

diff --git a/src/tabular/feature_engineering/new_featureEngeneer/discretizer.py b/src/tabular/feature_engineering/new_featureEngeneer/discretizer.py
index ba8d64b..efe3362 100644
--- a/src/tabular/feature_engineering/new_featureEngeneer/discretizer.py
+++ b/src/tabular/feature_engineering/new_featureEngeneer/discretizer.py
@@ -5,51 +5,8 @@
 
 
 def discretizer(df_list, names, method_list):
-    """
-    Parameters
-    ----------
-    df_list: pd.DataFrame type, the dataframe need to split.
-    names: list of column names
-    method_list: a dictionary contains the methods as key, and parameters as values.
-    key must in ['isometric','quantile','kmeans']
-    Like {"isometric":[n_bins]},{"quantile":[n_bins]},{"kmeans":[n_bins]}
+   """
 
-    Returns
-    -------
-    discretizers : list type. the discretizers of all column
-    data: the np.array after trans
-    """
-    if names is None:
-        data = check_array(df_list)
-    else:
-        data = check_array(df_list[names])
 
-    if len(method_list.keys) > 1:
-        raise ValueError("method_list only can has 1 key")
-
-    method = list(method_list.keys)[0]
-    if method not in ("isometric", "quantile", "kmeans"):
-        raise ValueError("`method` must be 'isometric','quantile' or 'kmeans'")
-
-    discretizers = []
-    if method == "isometric":
-        for column in range(np.shape(data)[0]):
-            discretizer = KBinsDiscretizer(n_bins=method_list[method], encode="ordinal", strategy="uniform")
-            fit_encoder(column, data, discretizer, discretizers)
-
-    elif method == "quantile":
-        for column in range(np.shape(data)[0]):
-            discretizer = KBinsDiscretizer(n_bins=method_list[method], encode="ordinal", strategy="quantile")
-            fit_encoder(column, data, discretizer, discretizers)
-    else:
-        for column in range(np.shape(data)[0]):
-            discretizer = KBinsDiscretizer(n_bins=method_list[method], encode="ordinal", strategy="kmeans")
-            fit_encoder(column, data, discretizer, discretizers)
-
-    return discretizers, data
-
-
-def fit_encoder(column, data, discretizer, discretizers):
-    discretizer.fit(data[:, column])
-    data[:, column] = discretizer.transform(data[:, column])
-    discretizers.append(discretizer)
+   """
+   pass
\ No newline at end of file

From ebe1726ee4787a2cdc828597fda1f6797ad0c373 Mon Sep 17 00:00:00 2001
From: sugangnb <sugangsky@gmail.com>
Date: Wed, 8 Jan 2020 12:01:50 +0800
Subject: [PATCH 2/4] discretizer comment update

---
 .../new_featureEngeneer/discretizer.py        | 70 +++++++++++++++++--
 .../new_featureEngeneer/feature_across.py     | 22 ------
 2 files changed, 65 insertions(+), 27 deletions(-)
 delete mode 100644 src/tabular/feature_engineering/new_featureEngeneer/feature_across.py

diff --git a/src/tabular/feature_engineering/new_featureEngeneer/discretizer.py b/src/tabular/feature_engineering/new_featureEngeneer/discretizer.py
index efe3362..7cb58c5 100644
--- a/src/tabular/feature_engineering/new_featureEngeneer/discretizer.py
+++ b/src/tabular/feature_engineering/new_featureEngeneer/discretizer.py
@@ -1,12 +1,72 @@
 # coding = 'utf-8'
 import numpy as np
-from sklearn.utils import check_array
+import pandas as pd
 from sklearn.preprocessing import KBinsDiscretizer
 
 
-def discretizer(df_list, names, method_list):
-   """
+class dis_configure:
+    """
+    The config object of discretizer. It saves the parameters of discretizer and check their validity.
 
+    Parameters
+    ----------
+    method : {'uniform', 'quantile', 'kmeans'}, (default='quantile')
+        uniform
+            All bins in each feature have identical widths.
+        quantile
+            All bins in each feature have the same number of points.
+        kmeans
+            Values in each bin have the same nearest center of a 1D k-means
+            cluster.
+
+    n_bins : int, default=5
+    """
+
+    method = None
+    n_bins = None
+
+    def _check(self):
+        if self.method is None:
+            self.method = "quantile"
+        elif self.method not in ['uniform', 'quantile', 'kmeans']:
+            raise ValueError(
+                "the method value {} is Invalid! It must be in ['uniform', 'quantile', 'kmeans'], "
+                "default is 'quantile'".format(
+                    str(self.method)))
+
+        if self.n_bins is None:
+            self.n_bins = 5
+        elif not isinstance(self.n_bins, int):
+            raise ValueError(
+                "the n_bins value {} is Invalid! It must be Int value "
+                "default is 5".format(
+                    str(self.n_bins)))
+
+    def __init__(self, method, n_bins):
+        self.method = method
+        self.n_bins = n_bins
+        self._check()
+
+
+def discretizer(df_list, names, config):
+    """
+    concat the df_list as one pd.DataFrame then using sklean.KBinsDiscretizer depart it,
+
+    Parameters
+    ----------
+    df_list : object
+        a collection of one or more pd.DataFrame. they must have one column named 'id' for indexing.
+    names : list
+        a list of the continuous variable column's name. If it's none,checking if all columns are continuous and
+        discrete the continuous variable columns.
+ 
+    config : object
+        the object of parameters
+
+    Returns
+    ----------
+    df_list_t : object
+        the df_list after trans ,still is the collection of pd.DataFrame
+    """
+    pass
 
-   """
-   pass
\ No newline at end of file
diff --git a/src/tabular/feature_engineering/new_featureEngeneer/feature_across.py b/src/tabular/feature_engineering/new_featureEngeneer/feature_across.py
deleted file mode 100644
index 536e5e9..0000000
--- a/src/tabular/feature_engineering/new_featureEngeneer/feature_across.py
+++ /dev/null
@@ -1,22 +0,0 @@
-import numpy as np
-
-def featureAcross(df_list, names, methods_list):
-    '''
-    连续变量的分箱、决策树组合特征
-    :param df_list: 
-    :param methods_list: {'bin':{'names':'houseArea', 'bin_dot':[100,500,1000]},
-                            }
-    :return: 
-    '''
-    if method not in ('bin','decision tree'):
-        raise ValueError('Input a method in ('bin','decision tree')')
-
-    if method == 'bin':
-
-
-
-
-def get_quantile_based_boundaries(feature_values, num_buckets):
-    boundaries = np.arange(1.0, num_buckets) / num_buckets
-    quantiles = feature_values.quantile(boundaries)
-    return [quantiles[q] for q in quantiles.keys()]

From c66ba95c59f20c0e18adee3061d2d9627d5c7b75 Mon Sep 17 00:00:00 2001
From: sugangnb <sugangsky@gmail.com>
Date: Thu, 9 Jan 2020 16:17:43 +0800
Subject: [PATCH 3/4] discretizer update & utils create

---
 .../new_featureEngeneer/discretizer.py        | 111 +++++++++++++++++-
 .../new_featureEngeneer/utils.py              |  20 ++++
 2 files changed, 128 insertions(+), 3 deletions(-)
 create mode 100644 src/tabular/feature_engineering/new_featureEngeneer/utils.py

diff --git a/src/tabular/feature_engineering/new_featureEngeneer/discretizer.py b/src/tabular/feature_engineering/new_featureEngeneer/discretizer.py
index 7cb58c5..84fdf4d 100644
--- a/src/tabular/feature_engineering/new_featureEngeneer/discretizer.py
+++ b/src/tabular/feature_engineering/new_featureEngeneer/discretizer.py
@@ -1,7 +1,9 @@
 # coding = 'utf-8'
-import numpy as np
 import pandas as pd
+import warnings
+
 from sklearn.preprocessing import KBinsDiscretizer
+from .utils import get_continue_feature
 
 
 class dis_configure:
@@ -20,10 +22,13 @@ class dis_configure:
             cluster.
 
     n_bins : int, default=5
+    index_col : str, default='id'
+        the col of df_list's DataFrame index col
     """
 
     method = None
     n_bins = None
+    index_col = None
 
     def _check(self):
         if self.method is None:
@@ -42,12 +47,99 @@ def _check(self):
                 "default is 5".format(
                     str(self.n_bins)))
 
-    def __init__(self, method, n_bins):
+        if self.index_col is None:
+            self.index_col = "id"
+        elif not isinstance(self.index_col, str):
+            raise ValueError(
+                "the index_col value {} is Invalid! It must be str value "
+                "default is 'id'".format(
+                    str(self.index_col)))
+
+    def __init__(self, method, n_bins, index_col):
         self.method = method
         self.n_bins = n_bins
+        self.index_col = index_col
         self._check()
 
 
+def check_index_col(data, config):
+    """
+    check the index column's values are unique.
+
+    Parameters
+    ----------
+    data : pd.Dataframe
+    config : object,
+        the config parameter object.
+
+    Returns
+    ----------
+    data : pd.Dataframe,
+        origin data
+    """
+
+    index_col_data = data[config.index_col]
+    if index_col_data.shape[1] == index_col_data.drop_duplicates().shape[1]:
+        return data
+    else:
+        raise ValueError("the index column '{}' values must be unique".format(config.index_col))
+
+
+def retrun_df_list(df_list,data,config):
+    """
+
+    Parameters
+    ----------
+    df_list :object
+        a collection of one or more pd.DataFrame. they must have one column named 'id' for indexing.
+    data : pd.DataFrame
+        the DataFrame after discrete
+    config : Object
+        the object of parameters
+
+    Returns
+    -------
+    df_list_t :
+        a collection of one or more pd.DataFrame, they are transformed.
+    """
+    df_list_t = list()
+    if isinstance(df_list, tuple) or isinstance(df_list, list):
+        for i in range(len(df_list)):
+            df_list_t.append(df_list[i][[config.index_col]].merge(data,on=config.index_col))
+
+    elif isinstance(df_list, pd.DataFrame):
+       df_list_t.append(df_list[[config.index_col]].merge(data,on=config.index_col))
+    else:
+        raise ValueError("paramter df_list must be the collection of one or more pd.DataFrame")
+
+    return df_list_t
+
+def concat_df_list(df_list, config):
+    """
+    concat the df_list as one dataframe
+    Parameters
+    ----------
+    df_list : pd.DataFrame or collection of pd.DataFrame
+        the origin collection of pd.DataFrame
+    config : object
+        the object of parameters
+
+    Returns
+    -------
+    df
+
+    """
+    if isinstance(df_list, tuple) or isinstance(df_list, list):
+        data = pd.concat(df_list, axis=1)
+    elif isinstance(df_list, pd.DataFrame):
+        data = df_list
+    else:
+        raise ValueError("paramter df_list must be the collection of one or more pd.DataFrame")
+
+    df = check_index_col(data, config)
+    return df
+
+
 def discretizer(df_list, names, config):
     """
     concat the df_list as one pd.DataFrame then using sklean.KBinsDiscretizer depart it,
@@ -68,5 +160,18 @@ def discretizer(df_list, names, config):
     df_list_t : object
         the df_list after trans ,still is the collection of pd.DataFrame
     """
-    pass
+    data = concat_df_list(df_list, config)
+
+    if names is None:
+        warnings.warn("The parameter names is None, will check th")
+        names, _ = get_continue_feature(data)
+
+    for name in names:
+        kbdis = KBinsDiscretizer(n_bins=config.n_bins,encode="ordinal",strategy=config.method)
+        kbdis.fit(data[name])
+        data.loc[:,name+"_discred"]=kbdis.transform(data[name])
+
+    return retrun_df_list(df_list,data,config)
+
+
 
diff --git a/src/tabular/feature_engineering/new_featureEngeneer/utils.py b/src/tabular/feature_engineering/new_featureEngeneer/utils.py
new file mode 100644
index 0000000..4b35cb3
--- /dev/null
+++ b/src/tabular/feature_engineering/new_featureEngeneer/utils.py
@@ -0,0 +1,20 @@
+# encoding:utf-8
+"""
+@author: sugang
+@time: 2020/1/9 3:38 下午
+@desc: 
+"""
+import pandas as pd
+
+def get_continue_feature(data):
+    continus_features, discrete_features = [], []
+    for col in data.columns:
+        if data[col].dtype != object:
+            try:
+                pd.qcut(data[col], 4)
+                continus_features.append(col)
+            except ValueError:
+                discrete_features.append(col)
+    print('Continus features are: ', continus_features)
+    print('Discrete features are: ', discrete_features)
+    return continus_features, discrete_features

From 88ff8aafc87d75726de8ef8b260bd3febba31c73 Mon Sep 17 00:00:00 2001
From: sugangnb <sugangsky@gmail.com>
Date: Fri, 10 Jan 2020 13:28:55 +0800
Subject: [PATCH 4/4] 	modified:  
 src/tabular/feature_engineering/new_featureEngeneer/discretizer.py 
 modified:   src/tabular/feature_engineering/new_featureEngeneer/utils.py

---
 .../feature_engineering/new_featureEngeneer/discretizer.py  | 1 -
 .../feature_engineering/new_featureEngeneer/utils.py        | 6 +-----
 2 files changed, 1 insertion(+), 6 deletions(-)

diff --git a/src/tabular/feature_engineering/new_featureEngeneer/discretizer.py b/src/tabular/feature_engineering/new_featureEngeneer/discretizer.py
index 84fdf4d..6f58e3b 100644
--- a/src/tabular/feature_engineering/new_featureEngeneer/discretizer.py
+++ b/src/tabular/feature_engineering/new_featureEngeneer/discretizer.py
@@ -5,7 +5,6 @@
 from sklearn.preprocessing import KBinsDiscretizer
 from .utils import get_continue_feature
 
-
 class dis_configure:
     """
     The config object of discretizer. It saves the parameters of discretizer and check their validity.
diff --git a/src/tabular/feature_engineering/new_featureEngeneer/utils.py b/src/tabular/feature_engineering/new_featureEngeneer/utils.py
index 4b35cb3..a3cbda5 100644
--- a/src/tabular/feature_engineering/new_featureEngeneer/utils.py
+++ b/src/tabular/feature_engineering/new_featureEngeneer/utils.py
@@ -1,9 +1,5 @@
 # encoding:utf-8
-"""
-@author: sugang
-@time: 2020/1/9 3:38 下午
-@desc: 
-"""
+
 import pandas as pd
 
 def get_continue_feature(data):