From 506b4c78749041cb41cf65cba4f7cfdf52d38b76 Mon Sep 17 00:00:00 2001 From: sugangnb Date: Tue, 21 Jan 2020 16:05:39 +0800 Subject: [PATCH 1/3] new file: src/tabular/feature_engineering/gplearn.py deleted: src/tabular/feature_engineering/new_featureEngeneer/discretizer(1).py modified: src/common/pytorch/network/dnn/mlp.py modified: src/tabular/feature_engineering/gplearn.py --- src/tabular/feature_engineering/gplearn.py | 0 .../new_featureEngeneer/discretizer(1).py | 176 ------------------ 2 files changed, 176 deletions(-) create mode 100644 src/tabular/feature_engineering/gplearn.py delete mode 100644 src/tabular/feature_engineering/new_featureEngeneer/discretizer(1).py diff --git a/src/tabular/feature_engineering/gplearn.py b/src/tabular/feature_engineering/gplearn.py new file mode 100644 index 0000000..e69de29 diff --git a/src/tabular/feature_engineering/new_featureEngeneer/discretizer(1).py b/src/tabular/feature_engineering/new_featureEngeneer/discretizer(1).py deleted file mode 100644 index 6f58e3b..0000000 --- a/src/tabular/feature_engineering/new_featureEngeneer/discretizer(1).py +++ /dev/null @@ -1,176 +0,0 @@ -# coding = 'utf-8' -import pandas as pd -import warnings - -from sklearn.preprocessing import KBinsDiscretizer -from .utils import get_continue_feature - -class dis_configure: - """ - The config object of discretizer. It saves the parameters of discretizer and check their validity. - - Parameters - ---------- - method : {'uniform', 'quantile', 'kmeans'}, (default='quantile') - uniform - All bins in each feature have identical widths. - quantile - All bins in each feature have the same number of points. - kmeans - Values in each bin have the same nearest center of a 1D k-means - cluster. - - n_bins : int, default=5 - index_col : str, default='id' - the col of df_list's DataFrame index col - """ - - method = None - n_bins = None - index_col = None - - def _check(self): - if self.method is None: - self.method = "quantile" - elif self.method not in ['uniform', 'quantile', 'kmeans']: - raise ValueError( - "the method value {} is Invalid! It must be in ['uniform', 'quantile', 'kmeans'], " - "default is 'quantile'".format( - str(self.method))) - - if self.n_bins is None: - self.n_bins = 5 - elif not isinstance(self.n_bins, int): - raise ValueError( - "the n_bins value {} is Invalid! It must be Int value " - "default is 5".format( - str(self.n_bins))) - - if self.index_col is None: - self.index_col = "id" - elif not isinstance(self.index_col, str): - raise ValueError( - "the index_col value {} is Invalid! It must be str value " - "default is 'id'".format( - str(self.index_col))) - - def __init__(self, method, n_bins, index_col): - self.method = method - self.n_bins = n_bins - self.index_col = index_col - self._check() - - -def check_index_col(data, config): - """ - check the index column's values are unique. - - Parameters - ---------- - data : pd.Dataframe - config : object, - the config parameter object. - - Returns - ---------- - data : pd.Dataframe, - origin data - """ - - index_col_data = data[config.index_col] - if index_col_data.shape[1] == index_col_data.drop_duplicates().shape[1]: - return data - else: - raise ValueError("the index column '{}' values must be unique".format(config.index_col)) - - -def retrun_df_list(df_list,data,config): - """ - - Parameters - ---------- - df_list :object - a collection of one or more pd.DataFrame. they must have one column named 'id' for indexing. - data : pd.DataFrame - the DataFrame after discrete - config : Object - the object of parameters - - Returns - ------- - df_list_t : - a collection of one or more pd.DataFrame, they are transformed. - """ - df_list_t = list() - if isinstance(df_list, tuple) or isinstance(df_list, list): - for i in range(len(df_list)): - df_list_t.append(df_list[i][[config.index_col]].merge(data,on=config.index_col)) - - elif isinstance(df_list, pd.DataFrame): - df_list_t.append(df_list[[config.index_col]].merge(data,on=config.index_col)) - else: - raise ValueError("paramter df_list must be the collection of one or more pd.DataFrame") - - return df_list_t - -def concat_df_list(df_list, config): - """ - concat the df_list as one dataframe - Parameters - ---------- - df_list : pd.DataFrame or collection of pd.DataFrame - the origin collection of pd.DataFrame - config : object - the object of parameters - - Returns - ------- - df - - """ - if isinstance(df_list, tuple) or isinstance(df_list, list): - data = pd.concat(df_list, axis=1) - elif isinstance(df_list, pd.DataFrame): - data = df_list - else: - raise ValueError("paramter df_list must be the collection of one or more pd.DataFrame") - - df = check_index_col(data, config) - return df - - -def discretizer(df_list, names, config): - """ - concat the df_list as one pd.DataFrame then using sklean.KBinsDiscretizer depart it, - - Parameters - ---------- - df_list : object - a collection of one or more pd.DataFrame. they must have one column named 'id' for indexing. - names : list - a list of the continuous variable column's name. If it's none,checking if all columns are continuous and - discrete the continuous variable columns. -  - config : object - the object of parameters - - Returns - ---------- - df_list_t : object - the df_list after trans ,still is the collection of pd.DataFrame - """ - data = concat_df_list(df_list, config) - - if names is None: - warnings.warn("The parameter names is None, will check th") - names, _ = get_continue_feature(data) - - for name in names: - kbdis = KBinsDiscretizer(n_bins=config.n_bins,encode="ordinal",strategy=config.method) - kbdis.fit(data[name]) - data.loc[:,name+"_discred"]=kbdis.transform(data[name]) - - return retrun_df_list(df_list,data,config) - - - From 8f3c22d62e206a6ea32176cf192ac759c0c9b2bb Mon Sep 17 00:00:00 2001 From: sugangnb Date: Tue, 21 Jan 2020 16:42:36 +0800 Subject: [PATCH 2/3] deleted: src/tabular/feature_engineering/gplearn.py new file: src/tabular/feature_engineering/new_featureEngeneer/__init__.py new file: src/tabular/feature_engineering/new_featureEngeneer/gplearn.py modified: src/common/pytorch/network/dnn/mlp.py modified: src/tabular/feature_engineering/new_featureEngeneer/__init__.py modified: src/tabular/feature_engineering/new_featureEngeneer/discretizer.py modified: src/tabular/feature_engineering/new_featureEngeneer/gplearn.py modified: src/tabular/feature_engineering/new_featureEngeneer/utils.py --- src/tabular/feature_engineering/gplearn.py | 0 .../new_featureEngeneer/__init__.py | 6 +++ .../new_featureEngeneer/gplearn.py | 44 +++++++++++++++++++ 3 files changed, 50 insertions(+) delete mode 100644 src/tabular/feature_engineering/gplearn.py create mode 100644 src/tabular/feature_engineering/new_featureEngeneer/__init__.py create mode 100644 src/tabular/feature_engineering/new_featureEngeneer/gplearn.py diff --git a/src/tabular/feature_engineering/gplearn.py b/src/tabular/feature_engineering/gplearn.py deleted file mode 100644 index e69de29..0000000 diff --git a/src/tabular/feature_engineering/new_featureEngeneer/__init__.py b/src/tabular/feature_engineering/new_featureEngeneer/__init__.py new file mode 100644 index 0000000..5600a8f --- /dev/null +++ b/src/tabular/feature_engineering/new_featureEngeneer/__init__.py @@ -0,0 +1,6 @@ +# encoding:utf-8 +""" +@author: sugang +@time: 2020/1/21 4:26 下午 +@desc: +""" \ No newline at end of file diff --git a/src/tabular/feature_engineering/new_featureEngeneer/gplearn.py b/src/tabular/feature_engineering/new_featureEngeneer/gplearn.py new file mode 100644 index 0000000..70a86c6 --- /dev/null +++ b/src/tabular/feature_engineering/new_featureEngeneer/gplearn.py @@ -0,0 +1,44 @@ +# encoding:utf-8 +""" +@author: sugang +@time: 2020/1/21 3:15 下午 +@desc: +""" +import pandas as pd +from sklearn import datasets +from gplearn.genetic import SymbolicTransformer +from sklearn.model_selection import train_test_split + + + +class GPConfig: + def __init__(self, generation=1000, population_size=5000, hall_of_fame=100, n_components=10, + parsimony_coefficient=0.0005, max_samples=0.9): + self.generation = generation + self.population_size = population_size + self.hall_of_fame = hall_of_fame + self.n_components = n_components + self.parsimony_coefficient = parsimony_coefficient + self.max_samples = max_samples + self.function_set = ['add', 'sub', 'mul', 'div', 'log', 'sqrt', 'abs', 'neg', 'max', 'min'] + + +def gp_feature_generator(df_train, df_test, y_name, var_list, gp_config): + gp = SymbolicTransformer(generations=gp_config.generation, population_size=gp_config.population_size, + hall_of_fame=gp_config.hall_of_fame, n_components=gp_config.n_components, + function_set=gp_config.function_set, + parsimony_coefficient=gp_config.parsimony_coefficient, + max_samples=gp_config.max_samples, verbose=1, + random_state=0, n_jobs=3) + + y_train = df_train[[y_name]] + if var_list is None: + X_train = df_train.drop(y_name, axis=1) + X_test = df_test.drop(y_name, axis=1) + else: + X_train = df_train[var_list] + X_test = df_train[var_list] + + gp.fit(X_train, y_train) + + return gp.transform(X_test) From 549e015ab4e4b0faecf6093b2169e2ce0f9518a5 Mon Sep 17 00:00:00 2001 From: sugangnb Date: Wed, 22 Jan 2020 15:22:39 +0800 Subject: [PATCH 3/3] deleted: src/tabular/feature_engineering/new_featureEngeneer/mean_encoder.py modified: src/common/pytorch/network/dnn/mlp.py modified: src/tabular/feature_engineering/new_featureEngeneer/__init__.py modified: src/tabular/feature_engineering/new_featureEngeneer/discretizer.py modified: src/tabular/feature_engineering/new_featureEngeneer/gplearn.py modified: src/tabular/feature_engineering/new_featureEngeneer/keyValue_pairs.py modified: src/tabular/feature_engineering/new_featureEngeneer/utils.py --- .../new_featureEngeneer/mean_encoder.py | 21 ------------------- 1 file changed, 21 deletions(-) delete mode 100644 src/tabular/feature_engineering/new_featureEngeneer/mean_encoder.py diff --git a/src/tabular/feature_engineering/new_featureEngeneer/mean_encoder.py b/src/tabular/feature_engineering/new_featureEngeneer/mean_encoder.py deleted file mode 100644 index fdb3c39..0000000 --- a/src/tabular/feature_engineering/new_featureEngeneer/mean_encoder.py +++ /dev/null @@ -1,21 +0,0 @@ -import numpy as np - - -def mean_encoder(df_list, encoded_vars, target, method): - ''' - :param df_list: - :param encoded_vars: 需要编码的变量 - :param target: 预测变量 - :param method: mean or var - :return: - ''' - for A in encoded_vars: - if A not in df_list.cloumns or target not in df_list.cloumns: - raise TypeError('input valid field') - - if method == 'mean': - return df_list[A].groupby(target).aggregate(np.mean) - - elif method == 'var': - return df_list[A].groupby(target).aggregate(np.var) -