diff --git a/src/tabular/feature_engineering/new_featureEngeneer/discretizer.py b/src/tabular/feature_engineering/new_featureEngeneer/discretizer.py index ba8d64b..6f58e3b 100644 --- a/src/tabular/feature_engineering/new_featureEngeneer/discretizer.py +++ b/src/tabular/feature_engineering/new_featureEngeneer/discretizer.py @@ -1,55 +1,176 @@ # coding = 'utf-8' -import numpy as np -from sklearn.utils import check_array +import pandas as pd +import warnings + from sklearn.preprocessing import KBinsDiscretizer +from .utils import get_continue_feature + +class dis_configure: + """ + The config object of discretizer. It saves the parameters of discretizer and check their validity. + + Parameters + ---------- + method : {'uniform', 'quantile', 'kmeans'}, (default='quantile') + uniform + All bins in each feature have identical widths. + quantile + All bins in each feature have the same number of points. + kmeans + Values in each bin have the same nearest center of a 1D k-means + cluster. + + n_bins : int, default=5 + index_col : str, default='id' + the col of df_list's DataFrame index col + """ + + method = None + n_bins = None + index_col = None + + def _check(self): + if self.method is None: + self.method = "quantile" + elif self.method not in ['uniform', 'quantile', 'kmeans']: + raise ValueError( + "the method value {} is Invalid! It must be in ['uniform', 'quantile', 'kmeans'], " + "default is 'quantile'".format( + str(self.method))) + + if self.n_bins is None: + self.n_bins = 5 + elif not isinstance(self.n_bins, int): + raise ValueError( + "the n_bins value {} is Invalid! It must be Int value " + "default is 5".format( + str(self.n_bins))) + + if self.index_col is None: + self.index_col = "id" + elif not isinstance(self.index_col, str): + raise ValueError( + "the index_col value {} is Invalid! It must be str value " + "default is 'id'".format( + str(self.index_col))) + + def __init__(self, method, n_bins, index_col): + self.method = method + self.n_bins = n_bins + self.index_col = index_col + self._check() -def discretizer(df_list, names, method_list): +def check_index_col(data, config): """ + check the index column's values are unique. + Parameters ---------- - df_list: pd.DataFrame type, the dataframe need to split. - names: list of column names - method_list: a dictionary contains the methods as key, and parameters as values. - key must in ['isometric','quantile','kmeans'] - Like {"isometric":[n_bins]},{"quantile":[n_bins]},{"kmeans":[n_bins]} + data : pd.Dataframe + config : object, + the config parameter object. + + Returns + ---------- + data : pd.Dataframe, + origin data + """ + + index_col_data = data[config.index_col] + if index_col_data.shape[1] == index_col_data.drop_duplicates().shape[1]: + return data + else: + raise ValueError("the index column '{}' values must be unique".format(config.index_col)) + + +def retrun_df_list(df_list,data,config): + """ + + Parameters + ---------- + df_list :object + a collection of one or more pd.DataFrame. they must have one column named 'id' for indexing. + data : pd.DataFrame + the DataFrame after discrete + config : Object + the object of parameters Returns ------- - discretizers : list type. the discretizers of all column - data: the np.array after trans + df_list_t : + a collection of one or more pd.DataFrame, they are transformed. """ - if names is None: - data = check_array(df_list) + df_list_t = list() + if isinstance(df_list, tuple) or isinstance(df_list, list): + for i in range(len(df_list)): + df_list_t.append(df_list[i][[config.index_col]].merge(data,on=config.index_col)) + + elif isinstance(df_list, pd.DataFrame): + df_list_t.append(df_list[[config.index_col]].merge(data,on=config.index_col)) else: - data = check_array(df_list[names]) + raise ValueError("paramter df_list must be the collection of one or more pd.DataFrame") - if len(method_list.keys) > 1: - raise ValueError("method_list only can has 1 key") + return df_list_t - method = list(method_list.keys)[0] - if method not in ("isometric", "quantile", "kmeans"): - raise ValueError("`method` must be 'isometric','quantile' or 'kmeans'") +def concat_df_list(df_list, config): + """ + concat the df_list as one dataframe + Parameters + ---------- + df_list : pd.DataFrame or collection of pd.DataFrame + the origin collection of pd.DataFrame + config : object + the object of parameters - discretizers = [] - if method == "isometric": - for column in range(np.shape(data)[0]): - discretizer = KBinsDiscretizer(n_bins=method_list[method], encode="ordinal", strategy="uniform") - fit_encoder(column, data, discretizer, discretizers) + Returns + ------- + df - elif method == "quantile": - for column in range(np.shape(data)[0]): - discretizer = KBinsDiscretizer(n_bins=method_list[method], encode="ordinal", strategy="quantile") - fit_encoder(column, data, discretizer, discretizers) + """ + if isinstance(df_list, tuple) or isinstance(df_list, list): + data = pd.concat(df_list, axis=1) + elif isinstance(df_list, pd.DataFrame): + data = df_list else: - for column in range(np.shape(data)[0]): - discretizer = KBinsDiscretizer(n_bins=method_list[method], encode="ordinal", strategy="kmeans") - fit_encoder(column, data, discretizer, discretizers) + raise ValueError("paramter df_list must be the collection of one or more pd.DataFrame") + + df = check_index_col(data, config) + return df + + +def discretizer(df_list, names, config): + """ + concat the df_list as one pd.DataFrame then using sklean.KBinsDiscretizer depart it, + + Parameters + ---------- + df_list : object + a collection of one or more pd.DataFrame. they must have one column named 'id' for indexing. + names : list + a list of the continuous variable column's name. If it's none,checking if all columns are continuous and + discrete the continuous variable columns. +  + config : object + the object of parameters + + Returns + ---------- + df_list_t : object + the df_list after trans ,still is the collection of pd.DataFrame + """ + data = concat_df_list(df_list, config) + + if names is None: + warnings.warn("The parameter names is None, will check th") + names, _ = get_continue_feature(data) + + for name in names: + kbdis = KBinsDiscretizer(n_bins=config.n_bins,encode="ordinal",strategy=config.method) + kbdis.fit(data[name]) + data.loc[:,name+"_discred"]=kbdis.transform(data[name]) + + return retrun_df_list(df_list,data,config) - return discretizers, data -def fit_encoder(column, data, discretizer, discretizers): - discretizer.fit(data[:, column]) - data[:, column] = discretizer.transform(data[:, column]) - discretizers.append(discretizer) diff --git a/src/tabular/feature_engineering/new_featureEngeneer/utils.py b/src/tabular/feature_engineering/new_featureEngeneer/utils.py new file mode 100644 index 0000000..a3cbda5 --- /dev/null +++ b/src/tabular/feature_engineering/new_featureEngeneer/utils.py @@ -0,0 +1,16 @@ +# encoding:utf-8 + +import pandas as pd + +def get_continue_feature(data): + continus_features, discrete_features = [], [] + for col in data.columns: + if data[col].dtype != object: + try: + pd.qcut(data[col], 4) + continus_features.append(col) + except ValueError: + discrete_features.append(col) + print('Continus features are: ', continus_features) + print('Discrete features are: ', discrete_features) + return continus_features, discrete_features