diff --git a/.gitignore b/.gitignore
index 2813db8..275ec49 100644
--- a/.gitignore
+++ b/.gitignore
@@ -15,3 +15,4 @@ rgforest/rgfpackage/test
rgfpackage/bin/
rgfpackage/test/
htmlcov/
+.idea
\ No newline at end of file
diff --git a/.travis.yml b/.travis.yml
new file mode 100644
index 0000000..92ecd14
--- /dev/null
+++ b/.travis.yml
@@ -0,0 +1,17 @@
+language: python
+python:
+- '3.7'
+- '3.8'
+- '3.9'
+install:
+- pip install -r requirements.txt
+- pip install .
+script:
+- pytest -v
+deploy:
+ provider: pypi
+ username: __token__
+ password:
+ secure: qoKQp5jbF9iJs/vmTB//5mPrjOtcCc5zouED3Qo9Ws6RTl+WRTkdpCsufZDSflfjX1hgsOkySYtbE6A3DyyGPEyTnkc91sp5YPGCNApqiCl7Y7l5LKrmqqw99LXB+h+9luz2YzgDPY6oYiPbNhQuksWk8jj3QNWSBAwHRhj5Qul96BvBQYm/a0jXEJF+WRh2q+tSj6EwNrNHhrD7Rs8YMRkHnx1vsz5PnegvqajIfUp7QWx8XiRSBCR+x2cLU/QivtakAZITZ5cJ/WeX2XRmczFSmJZ0zC4KnytASwr56wnhwCOT0oNmh413E8IZ6a/TTffN4wcXZhZdRoAAGuy9fqKWXoPcuq+72kjwPTM6SMW1FvBt+Bo94y5rAh/eXC25M0TTqHsdPpJJa/14nA993xtIXAer4VOk3UNmc6ioLYkrOlvU85XkpO076Ll8NQfzseJMsCb7FZjQ5xy48TLGqOFYA9qBDnVBf2/BHAfjVbBL7XRjJCJOFCqhVmM48bxagoZhPDDzPc0uXhv8KGott2arAtlsgc91zStMHzM6COyJ5V3lIS/ccwY5QFjkbW8xzOwVlO0nHSNCYNmO4LFb29cNe/F645RNeGhNdMYED81sKG7Lh8dXlliqu5lslKRQBmNtV/gf4HdRnmbKldFYAWU0p2dJHj42kTRygVbvW7M=
+ on:
+ tags: true
diff --git a/README.md b/README.md
index 58c4d36..15b576a 100644
--- a/README.md
+++ b/README.md
@@ -1,22 +1,41 @@
Rotation Forest
-===============
-Simple hack to sklearn's random forest module to implement the Rotation Forest algorithm from Rodriguez et al. 2006.
+---
+[](https://travis-ci.org/digital-idiot/RotationForest) [](https://opensource.org/licenses/MIT)
+---
+Simple derived implementation of Rotation Forest algorithm[1, 2] based upon sklearn's random forest module.
Algorithm
----------
+---
+```
for tree in trees:
-- split the attributes in the training set into K non-overlapping subsets of equal size.
-- bootstrap 75% of the data from each K dataset and use the bootstrap data in the following steps.
-- Run PCA on the ith subset in K. Retain all principal components. For every feature j in the Kth subsets, we have a principal component a.
+ split the attributes in the training set into K non-overlapping subsets of equal size.
+ bootstrap þ% of the data from each K dataset and use the bootstrap data in the following steps:
+ - Run PCA on the i-th subset in K. Retain all principal components. For every feature j in the Kth subsets, we have a principal component a.
+ - Create a rotation matrix of size n X n where n is the total number of features. Arrange the principal component in the matrix such that the components match the position of the feature in the original training dataset.
+ - Project the training dataset on the rotation matrix.
+ - Build a decision tree with the projected dataset
+ - Store the tree and the rotation matrix.
+```
-Create a rotation matrix of size n X n where n is the total number of features. Arrange
-the principal component in the matrix such that the components match the position of the feature in the original training dataset.
+* Rotation forest has been implemented both as classifier[1] and regressor[2].
-Project the training dataset on the rotation matrix.
-Build a decision tree with the projected dataset
+
+[1] J. J. Rodriguez, L. I. Kuncheva, and C. J. Alonso, “Rotation Forest: A New Classifier Ensemble Method,” IEEE Trans. Pattern Anal. Mach. Intell., vol. 28, no. 10, pp. 1619–1630, Oct. 2006, doi: 10.1109/tpami.2006.211.
+
-Store the tree and the rotation matrix.
+
+[2] C. Pardo, J. F. Diez-Pastor, C. García-Osorio, and J. J. Rodríguez, “Rotation Forests for regression,” Applied Mathematics and Computation, vol. 219, no. 19, pp. 9914–9924, Jun. 2013, doi: 10.1016/j.amc.2013.03.139.
+
-
+---
+
+## Toy Benchmark
+* ### Random Forest Classifier
+
+
+
+* ### Random Forest Regressor
+
+
diff --git a/benchmarks/benchmarks.py b/benchmarks/benchmarks.py
index 6b481fb..fd1f50b 100644
--- a/benchmarks/benchmarks.py
+++ b/benchmarks/benchmarks.py
@@ -1,13 +1,13 @@
#!/usr/bin/python
-import os
import time
import random
import numpy as np
+from pathlib import Path
-from sklearn import cross_validation
+# from sklearn import cross_validation
from sklearn.metrics import accuracy_score
-from sklearn.cross_validation import StratifiedKFold
+from sklearn.model_selection import StratifiedKFold
from sklearn.ensemble import RandomForestClassifier
from rotation_forest import RotationForestClassifier
@@ -52,22 +52,28 @@ def read_uci_dataset(path):
return X, Y
if __name__ == '__main__':
+ c_dir = Path('../')
+ dataset_dir = (c_dir / 'datasets') / 'classification_data'
+ print(c_dir.absolute())
+ for dataset in dataset_dir.glob('*.data'):
+ name = dataset.name.split('.')[0]
- dataset_dir = './datasets/'
- for dataset in os.listdir(dataset_dir):
- name = dataset.split('.')[0]
+ X, Y = read_uci_dataset(dataset)
- X, Y = read_uci_dataset(dataset_dir+dataset)
-
- K = 10
+ k = 10
accuracy = []
- cv = StratifiedKFold(Y, K)
+ skf = StratifiedKFold(
+ n_splits=k, shuffle=True, random_state=1234
+ )
- print name
+ print(name)
for clf in [RotationForestClassifier(), RandomForestClassifier(n_estimators=10)]:
- for train, test in cv:
+ #for train, test in cv:
+ for train_index, test_index in skf.split(X, Y):
- x_train, x_test, y_train, y_test = X[train,:], X[test,:], Y[train], Y[test]
+ #x_train, x_test, y_train, y_test = X[train,:], X[test,:], Y[train], Y[test]
+ x_train, x_test = X[train_index], X[test_index]
+ y_train, y_test = Y[train_index], Y[test_index]
clf = clf.fit(x_train, y_train)
y_pred = clf.predict(x_test)
@@ -75,5 +81,9 @@ def read_uci_dataset(path):
bd_std = np.std(accuracy)
bd_acc = np.mean(accuracy)
- print '{0}: {1:2f} +/- {2:2f}'.format(clf.__class__.__name__, bd_acc*100, bd_std*100)
- print ""
+ print(
+ '{0}: {1:2f} +/- {2:2f}'.format(
+ clf.__class__.__name__, bd_acc*100, bd_std*100
+ )
+ )
+ print()
diff --git a/benchmarks/simple_benchmark.png b/benchmarks/simple_benchmark.png
deleted file mode 100644
index 106c3d9..0000000
Binary files a/benchmarks/simple_benchmark.png and /dev/null differ
diff --git a/benchmarks/simple_benchmark_classification.png b/benchmarks/simple_benchmark_classification.png
new file mode 100644
index 0000000..2ce3ca8
Binary files /dev/null and b/benchmarks/simple_benchmark_classification.png differ
diff --git a/benchmarks/simple_benchmark.py b/benchmarks/simple_benchmark_classification.py
similarity index 56%
rename from benchmarks/simple_benchmark.py
rename to benchmarks/simple_benchmark_classification.py
index f182fca..4196c13 100644
--- a/benchmarks/simple_benchmark.py
+++ b/benchmarks/simple_benchmark_classification.py
@@ -5,7 +5,7 @@
from sklearn.ensemble import RandomForestClassifier, BaggingClassifier
from sklearn.ensemble import AdaBoostClassifier
from sklearn.tree import DecisionTreeClassifier
-from sklearn.cross_validation import StratifiedKFold
+from sklearn.model_selection import StratifiedKFold
from sklearn.metrics import roc_auc_score
from sklearn.pipeline import make_pipeline
@@ -20,20 +20,23 @@ def classification_data():
n_informative = int(0.6 * n_features)
n_repeated = int(0.1 * n_features)
- X, y = make_classification(n_samples=500, n_features=n_features, flip_y=0.03,
- n_informative=n_informative, n_redundant=n_redundant,
- n_repeated=n_repeated, random_state=9)
+ X, y = make_classification(
+ n_samples=500, n_features=n_features, flip_y=0.03,
+ n_informative=n_informative, n_redundant=n_redundant,
+ n_repeated=n_repeated, random_state=9
+ )
return X, y
def test_toy_data(name, clf):
X, y = classification_data()
k_folds = 5
- cv = StratifiedKFold(y, k_folds, random_state=1234)
+ skf = StratifiedKFold(n_splits=k_folds, shuffle=True, random_state=1234)
acc, auc = [], []
- for train, test in cv:
- xt, xv, yt, yv = X[train, :], X[test, :], y[train], y[test]
+ for train_index, test_index in skf.split(X, y):
+ xt, xv = X[train_index], X[test_index]
+ yt, yv = y[train_index], y[test_index]
clf.fit(xt, yt)
yhat = clf.predict(xv)
proba = clf.predict_proba(xv)[:, 1]
@@ -42,10 +45,10 @@ def test_toy_data(name, clf):
acc_mean, acc_std = np.mean(acc), np.std(acc)
auc_mean, auc_std = np.mean(auc), np.std(auc)
- print name
- print 'accuracy: {0:.3f} +/- {1:.3f}'.format(acc_mean, acc_std)
- print 'auc: {0:.3f} +/- {1:.3f}'.format(auc_mean, auc_std)
- print '-'*80
+ print(name)
+ print('accuracy: {0:.3f} +/- {1:.3f}'.format(acc_mean, acc_std))
+ print('auc: {0:.3f} +/- {1:.3f}'.format(auc_mean, auc_std))
+ print('-'*80)
return {'name': name,
'acc_mean': acc_mean,
'acc_std': acc_std,
@@ -55,30 +58,29 @@ def test_toy_data(name, clf):
classifiers = [('Random Forest',
RandomForestClassifier(random_state=12, n_estimators=25)),
('PCA + Random Forest',
- make_pipeline(PCA(), RandomForestClassifier(random_state=12,
- n_estimators=25))),
+ make_pipeline(PCA(), RandomForestClassifier(
+ random_state=12, n_estimators=25))),
('Rotation Tree',
RotationTreeClassifier(random_state=12,
n_features_per_subset=3)),
('Decision Tree',
DecisionTreeClassifier(random_state=12)),
- ('Rotation Forest (PCA)',
+ ('Rotation Forest\n(PCA)',
RotationForestClassifier(random_state=12,
n_estimators=25,
n_features_per_subset=3)),
- ('Rotation Forest (Randomized PCA)',
+ ('Rotation Forest\n(Randomized PCA)',
RotationForestClassifier(random_state=12,
n_estimators=25,
n_features_per_subset=3,
rotation_algo='randomized')),
- ('Adaboost (Rotation Tree)',
- AdaBoostClassifier(RotationTreeClassifier(n_features_per_subset=3,
- random_state=12,
- max_depth=3),
+ ('Adaboost\n(Rotation Tree)',
+ AdaBoostClassifier(RotationTreeClassifier(
+ n_features_per_subset=3, random_state=12, max_depth=3),
n_estimators=25,
random_state=12)
),
- ('Adaboost (Decision Tree)',
+ ('Adaboost\n(Decision Tree)',
AdaBoostClassifier(DecisionTreeClassifier(random_state=12,
max_depth=3),
n_estimators=25,
@@ -92,10 +94,35 @@ def test_toy_data(name, clf):
for name, clf in classifiers:
results.append(test_toy_data(name, clf))
df = pd.DataFrame(results)
- plt.figure(figsize=(15, 12))
- sns.barplot(x='name', y='auc_mean', data=df.sort('auc_mean'),
- xerr=df['auc_std'].values)
- plt.xticks(rotation=45)
- plt.xlabel('Classifier')
- plt.ylabel('AUC')
- plt.savefig('simple_benchmark.png')
+ df = df.sort_values('auc_mean')
+ fig = plt.figure(figsize=(15, 15))
+ n = len(df.index)
+ cm = plt.cm.get_cmap('rainbow', n)
+ bcolors = [cm(i) for i in range(n)]
+ ax = fig.add_subplot(111)
+ ax.bar(
+ x=(np.arange(n) - 0.15),
+ height=df['auc_mean'],
+ bottom=0,
+ align='center',
+ color=bcolors,
+ yerr=df['auc_std'],
+ width=0.25
+ )
+ ax.bar(
+ x=(np.arange(n) + 0.15),
+ height=df['acc_mean'],
+ bottom=0,
+ align='center',
+ color=bcolors,
+ yerr=df['acc_std'],
+ width=0.25
+ )
+ ax.set_title('Classification Benchmark')
+ ax.set_xlabel('Classifier')
+ ax.set_ylabel('AUC / Accuracy')
+ ax.set_xticks(np.arange(n))
+ ax.set_xticklabels(df['name'])
+ ax.set_ylim(bottom=0.5, top=1.0)
+ ax.tick_params(axis="x", rotation=45)
+ plt.savefig('simple_benchmark_classification.png')
diff --git a/benchmarks/simple_benchmark_regression.png b/benchmarks/simple_benchmark_regression.png
new file mode 100644
index 0000000..81c05da
Binary files /dev/null and b/benchmarks/simple_benchmark_regression.png differ
diff --git a/benchmarks/simple_benchmark_regression.py b/benchmarks/simple_benchmark_regression.py
new file mode 100644
index 0000000..dfc9100
--- /dev/null
+++ b/benchmarks/simple_benchmark_regression.py
@@ -0,0 +1,137 @@
+import matplotlib.pyplot as plt
+import numpy as np
+import pandas as pd
+import seaborn as sns
+from sklearn.datasets import make_regression
+from sklearn.decomposition import PCA
+from sklearn.ensemble import RandomForestRegressor, AdaBoostRegressor
+from sklearn.model_selection import KFold
+from sklearn.pipeline import make_pipeline
+from sklearn.tree import DecisionTreeRegressor
+
+from rotation_forest.rotation_forest import RotationTreeRegressor, RotationForestRegressor
+
+
+def regression_data():
+ n_features = 30
+ n_informative = int(0.6 * n_features)
+
+ X, y = make_regression(n_samples=500, n_features=n_features,
+ n_informative=n_informative, random_state=9)
+
+ return X, y
+
+
+def _test_toy_data(name, clf):
+ X, y = regression_data()
+ k_folds = 5
+ kf = KFold(n_splits=k_folds, shuffle=True, random_state=1234)
+
+ mse, mae = [], []
+ for train_index, test_index in kf.split(X, y):
+ xt, xv = X[train_index], X[test_index]
+ yt, yv = y[train_index], y[test_index]
+ clf.fit(xt, yt)
+ yhat = clf.predict(xv)
+ mse.append(np.mean((yhat - yv) ** 2))
+ mae.append(np.mean(np.abs((yhat - yv))))
+
+ mse_mean, mse_std = np.mean(mse), np.std(mse)
+ mae_mean, mae_std = np.mean(mae), np.std(mae)
+ print(name)
+ print('mse: {0:.3f} +/- {1:.3f}'.format(mse_mean, mse_std))
+ print('mae: {0:.3f} +/- {1:.3f}'.format(mae_mean, mae_std))
+ print('-' * 80)
+ return {'name': name,
+ 'mse_mean': mse_mean,
+ 'mse_std': mse_std,
+ 'mae_mean': mae_mean,
+ 'mae_std': mae_std}
+
+
+classifiers = [('Random Forest',
+ RandomForestRegressor(random_state=12, n_estimators=25)),
+ ('PCA + Random Forest',
+ make_pipeline(PCA(), RandomForestRegressor(random_state=12,
+ n_estimators=25))),
+ ('Rotation Tree',
+ RotationTreeRegressor(random_state=12,
+ n_features_per_subset=3)),
+ ('Decision Tree',
+ DecisionTreeRegressor(random_state=12)),
+ ('Rotation Forest\n(PCA)',
+ RotationForestRegressor(random_state=12,
+ n_estimators=25,
+ n_features_per_subset=3)),
+ ('Rotation Forest\n(Randomized PCA)',
+ RotationForestRegressor(random_state=12,
+ n_estimators=25,
+ n_features_per_subset=3,
+ rotation_algo='randomized')),
+ ('Adaboost\n(Rotation Tree)',
+ AdaBoostRegressor(RotationTreeRegressor(n_features_per_subset=3,
+ random_state=12,
+ max_depth=3),
+ n_estimators=25,
+ random_state=12)
+ ),
+ ('Adaboost\n(Decision Tree)',
+ AdaBoostRegressor(DecisionTreeRegressor(random_state=12,
+ max_depth=3),
+ n_estimators=25,
+ random_state=12)
+ )]
+
+if __name__ == '__main__':
+ results = []
+ for name, clf in classifiers:
+ results.append(_test_toy_data(name, clf))
+ df = pd.DataFrame(results)
+ df = df.sort_values('mse_mean')
+ fig = plt.figure(figsize=(20, 15))
+ n = len(df.index)
+ cm = plt.cm.get_cmap('rainbow', n)
+ bcolors = [cm(i) for i in range(n)]
+
+ ax1 = fig.add_subplot(121)
+ ax1.bar(
+ x=np.arange(n),
+ height=df['mse_mean'],
+ bottom=0,
+ align='center',
+ color=bcolors,
+ yerr=df['mse_std'],
+ width=0.25
+ )
+ ax1.set_xlabel('Classifier')
+ ax1.set_ylabel('MSE')
+ ax1.set_xticks(np.arange(n))
+ ax1.set_xticklabels(df['name'])
+ ax1.tick_params(axis="x", rotation=45)
+
+ ax2 = fig.add_subplot(122)
+ ax2.bar(
+ x=np.arange(n),
+ height=df['mae_mean'],
+ bottom=0,
+ align='center',
+ color=bcolors,
+ yerr=df['mae_std'],
+ width=0.25
+ )
+ ax2.set_xlabel('Classifier')
+ ax2.set_ylabel('MAE')
+ ax2.set_xticks(np.arange(n))
+ ax2.set_xticklabels(df['name'])
+ ax2.tick_params(axis="x", rotation=45)
+
+ fig.suptitle('Regression Benchmark')
+ plt.savefig('simple_benchmark_regression.png')
+
+ #plt.figure(figsize=(15, 15))
+ #sns.barplot(x='name', y='mse_mean', data=df.sort_values('mse_mean'),
+ #xerr=df['mse_std'].values)
+ #plt.xticks(rotation=45)
+ #plt.xlabel('Regressor')
+ #plt.ylabel('MSE')
+ #plt.savefig('simple_benchmark_regression.png')
diff --git a/datasets/Balance.data b/datasets/classification_data/Balance.data
similarity index 100%
rename from datasets/Balance.data
rename to datasets/classification_data/Balance.data
diff --git a/datasets/Breast-can.data b/datasets/classification_data/Breast-can.data
similarity index 100%
rename from datasets/Breast-can.data
rename to datasets/classification_data/Breast-can.data
diff --git a/datasets/Diabetes.data b/datasets/classification_data/Diabetes.data
similarity index 100%
rename from datasets/Diabetes.data
rename to datasets/classification_data/Diabetes.data
diff --git a/datasets/Digit.data b/datasets/classification_data/Digit.data
similarity index 100%
rename from datasets/Digit.data
rename to datasets/classification_data/Digit.data
diff --git a/datasets/Ecoli.data b/datasets/classification_data/Ecoli.data
similarity index 100%
rename from datasets/Ecoli.data
rename to datasets/classification_data/Ecoli.data
diff --git a/datasets/Hayes.data b/datasets/classification_data/Hayes.data
similarity index 100%
rename from datasets/Hayes.data
rename to datasets/classification_data/Hayes.data
diff --git a/datasets/Iris.data b/datasets/classification_data/Iris.data
similarity index 100%
rename from datasets/Iris.data
rename to datasets/classification_data/Iris.data
diff --git a/datasets/Liver.data b/datasets/classification_data/Liver.data
similarity index 100%
rename from datasets/Liver.data
rename to datasets/classification_data/Liver.data
diff --git a/datasets/Monk1.data b/datasets/classification_data/Monk1.data
similarity index 100%
rename from datasets/Monk1.data
rename to datasets/classification_data/Monk1.data
diff --git a/datasets/Monk2.data b/datasets/classification_data/Monk2.data
similarity index 100%
rename from datasets/Monk2.data
rename to datasets/classification_data/Monk2.data
diff --git a/datasets/Monk3.data b/datasets/classification_data/Monk3.data
similarity index 100%
rename from datasets/Monk3.data
rename to datasets/classification_data/Monk3.data
diff --git a/datasets/Sonar.data b/datasets/classification_data/Sonar.data
similarity index 100%
rename from datasets/Sonar.data
rename to datasets/classification_data/Sonar.data
diff --git a/datasets/Soybean.data b/datasets/classification_data/Soybean.data
similarity index 100%
rename from datasets/Soybean.data
rename to datasets/classification_data/Soybean.data
diff --git a/datasets/Spambase.data b/datasets/classification_data/Spambase.data
similarity index 100%
rename from datasets/Spambase.data
rename to datasets/classification_data/Spambase.data
diff --git a/datasets/Waveform.data b/datasets/classification_data/Waveform.data
similarity index 100%
rename from datasets/Waveform.data
rename to datasets/classification_data/Waveform.data
diff --git a/datasets/Wine.data b/datasets/classification_data/Wine.data
similarity index 100%
rename from datasets/Wine.data
rename to datasets/classification_data/Wine.data
diff --git a/requirements.txt b/requirements.txt
new file mode 100644
index 0000000..78e3b11
--- /dev/null
+++ b/requirements.txt
@@ -0,0 +1,34 @@
+# Automatically generated by https://github.com/damnever/pigar.
+
+# RotationForest/benchmarks/simple_benchmark.py: 14
+# RotationForest/benchmarks/simple_benchmark_regression.py: 1
+matplotlib ~= 3.3.4
+
+# RotationForest/benchmarks/benchmarks.py: 6
+# RotationForest/benchmarks/simple_benchmark.py: 1
+# RotationForest/benchmarks/simple_benchmark_regression.py: 2
+# RotationForest/rotation_forest/rotation_forest.py: 1
+# RotationForest/rotation_forest/tests/test_rotation_forest.py: 4,5
+numpy ~> 3.5.0
+
+# RotationForest/benchmarks/simple_benchmark.py: 2
+# RotationForest/benchmarks/simple_benchmark_regression.py: 3
+pandas ~= 1.2.2
+
+# RotationForest/rotation_forest/tests/test_rotation_forest.py: 1
+pytest ~= 6.2.2
+
+# RotationForest/benchmarks/benchmarks.py: 8,9,10,12
+# RotationForest/benchmarks/simple_benchmark.py: 5,6,7,8,9,11,12
+# RotationForest/benchmarks/simple_benchmark_regression.py: 7,8,9,10
+# RotationForest/rotation_forest/_exceptions.py: 19,21,81,83,85
+# RotationForest/rotation_forest/rotation_forest.py: 5
+# RotationForest/rotation_forest/tests/test_rotation_forest.py: 6,7
+scikit_learn ~> 1.5.0
+
+# RotationForest/benchmarks/simple_benchmark.py: 15
+# RotationForest/benchmarks/simple_benchmark_regression.py: 4
+seaborn ~= 0.11.1
+
+# RotationForest/setup.py: 1
+# setuptools ~= 49.6.0.post20210108
diff --git a/rotation_forest/__init__.py b/rotation_forest/__init__.py
index 99f871e..993e80e 100644
--- a/rotation_forest/__init__.py
+++ b/rotation_forest/__init__.py
@@ -2,5 +2,7 @@
from __future__ import unicode_literals
from __future__ import division
+from .rotation_forest import random_feature_subsets
from .rotation_forest import RotationTreeClassifier
from .rotation_forest import RotationForestClassifier
+from ._exceptions import NotFittedError
diff --git a/rotation_forest/rotation_forest.py b/rotation_forest/rotation_forest.py
index 8ed610e..0ae0f92 100644
--- a/rotation_forest/rotation_forest.py
+++ b/rotation_forest/rotation_forest.py
@@ -1,19 +1,15 @@
import numpy as np
-
-from sklearn.tree import DecisionTreeClassifier
-from sklearn.tree._tree import DTYPE
-from sklearn.ensemble.forest import ForestClassifier
+from sklearn.decomposition import PCA
+from sklearn.ensemble._forest import ForestClassifier, ForestRegressor
+from sklearn.tree import DecisionTreeClassifier, DecisionTreeRegressor
from sklearn.utils import resample, gen_batches, check_random_state
-from sklearn.utils.extmath import fast_dot
-from sklearn.decomposition import PCA, RandomizedPCA
-from _exceptions import NotFittedError
def random_feature_subsets(array, batch_size, random_state=1234):
""" Generate K subsets of the features in X """
random_state = check_random_state(random_state)
- features = range(array.shape[1])
- random_state.shuffle(features)
+ features = list(range(array.shape[1]))
+ random_state.shuffle([x for x in features])
for batch in gen_batches(len(features), batch_size):
yield features[batch]
@@ -32,7 +28,7 @@ def __init__(self,
random_state=None,
max_leaf_nodes=None,
class_weight=None,
- presort=False):
+ ):
self.n_features_per_subset = n_features_per_subset
self.rotation_algo = rotation_algo
@@ -48,18 +44,18 @@ def __init__(self,
max_leaf_nodes=max_leaf_nodes,
class_weight=class_weight,
random_state=random_state,
- presort=presort)
+ )
def rotate(self, X):
if not hasattr(self, 'rotation_matrix'):
raise NotFittedError('The estimator has not been fitted')
- return fast_dot(X, self.rotation_matrix)
+ return np.dot(X, self.rotation_matrix)
def pca_algorithm(self):
""" Deterimine PCA algorithm to use. """
if self.rotation_algo == 'randomized':
- return RandomizedPCA(random_state=self.random_state)
+ return PCA(svd_solver='randomized', random_state=self.random_state)
elif self.rotation_algo == 'pca':
return PCA()
else:
@@ -75,8 +71,8 @@ def _fit_rotation_matrix(self, X):
random_feature_subsets(X, self.n_features_per_subset,
random_state=self.random_state)):
# take a 75% bootstrap from the rows
- x_sample = resample(X, n_samples=int(n_samples*0.75),
- random_state=10*i)
+ x_sample = resample(X, n_samples=int(n_samples * 0.75),
+ random_state=10 * i)
pca = self.pca_algorithm()
pca.fit(x_sample[:, subset])
self.rotation_matrix[np.ix_(subset, subset)] = pca.components_
@@ -87,8 +83,8 @@ def fit(self, X, y, sample_weight=None, check_input=True):
sample_weight, check_input)
def predict_proba(self, X, check_input=True):
- return super(RotationTreeClassifier, self).predict_proba(self.rotate(X),
- check_input)
+ return super(RotationTreeClassifier, self).predict_proba(self.rotate(X),
+ check_input)
def predict(self, X, check_input=True):
return super(RotationTreeClassifier, self).predict(self.rotate(X),
@@ -102,6 +98,7 @@ def decision_path(self, X, check_input=True):
return super(RotationTreeClassifier, self).decision_path(self.rotate(X),
check_input)
+
class RotationForestClassifier(ForestClassifier):
def __init__(self,
n_estimators=10,
@@ -146,3 +143,128 @@ def __init__(self,
self.min_weight_fraction_leaf = min_weight_fraction_leaf
self.max_features = max_features
self.max_leaf_nodes = max_leaf_nodes
+
+
+class RotationTreeRegressor(DecisionTreeRegressor):
+ def __init__(self,
+ n_features_per_subset=3,
+ rotation_algo='pca',
+ criterion="mse",
+ splitter="best",
+ max_depth=None,
+ min_samples_split=2,
+ min_samples_leaf=1,
+ min_weight_fraction_leaf=0.,
+ max_features=1.0,
+ random_state=None,
+ max_leaf_nodes=None):
+
+ self.n_features_per_subset = n_features_per_subset
+ self.rotation_algo = rotation_algo
+
+ super(RotationTreeRegressor, self).__init__(
+ criterion=criterion,
+ splitter=splitter,
+ max_depth=max_depth,
+ min_samples_split=min_samples_split,
+ min_samples_leaf=min_samples_leaf,
+ min_weight_fraction_leaf=min_weight_fraction_leaf,
+ max_features=max_features,
+ max_leaf_nodes=max_leaf_nodes,
+ random_state=random_state)
+
+ def rotate(self, X):
+ if not hasattr(self, 'rotation_matrix'):
+ raise NotFittedError('The estimator has not been fitted')
+
+ return np.dot(X, self.rotation_matrix)
+
+ def pca_algorithm(self):
+ """ Deterimine PCA algorithm to use. """
+ if self.rotation_algo == 'randomized':
+ return PCA(svd_solver='randomized', random_state=self.random_state)
+ elif self.rotation_algo == 'pca':
+ return PCA()
+ else:
+ raise ValueError("`rotation_algo` must be either "
+ "'pca' or 'randomized'.")
+
+ def _fit_rotation_matrix(self, X):
+ random_state = check_random_state(self.random_state)
+ n_samples, n_features = X.shape
+ self.rotation_matrix = np.zeros((n_features, n_features),
+ dtype=np.float32)
+ for i, subset in enumerate(
+ random_feature_subsets(X, self.n_features_per_subset,
+ random_state=self.random_state)):
+ # take a 75% bootstrap from the rows
+ x_sample = resample(X, n_samples=int(n_samples * 0.75),
+ random_state=10 * i)
+ pca = self.pca_algorithm()
+ pca.fit(x_sample[:, subset])
+ self.rotation_matrix[np.ix_(subset, subset)] = pca.components_
+
+ def fit(self, X, y, sample_weight=None, check_input=True):
+ self._fit_rotation_matrix(X)
+ super(RotationTreeRegressor, self).fit(self.rotate(X), y,
+ sample_weight, check_input)
+
+ def predict_proba(self, X, check_input=True):
+ return super(RotationTreeRegressor, self).predict_proba(self.rotate(X),
+ check_input)
+
+ def predict(self, X, check_input=True):
+ return super(RotationTreeRegressor, self).predict(self.rotate(X),
+ check_input)
+
+ def apply(self, X, check_input=True):
+ return super(RotationTreeRegressor, self).apply(self.rotate(X),
+ check_input)
+
+ def decision_path(self, X, check_input=True):
+ return super(RotationTreeRegressor, self).decision_path(self.rotate(X),
+ check_input)
+
+
+class RotationForestRegressor(ForestRegressor):
+ def __init__(self,
+ n_estimators=10,
+ criterion="mse",
+ n_features_per_subset=3,
+ rotation_algo='pca',
+ max_depth=None,
+ min_samples_split=2,
+ min_samples_leaf=1,
+ min_weight_fraction_leaf=0.,
+ max_features=1.0,
+ max_leaf_nodes=None,
+ bootstrap=False,
+ oob_score=False,
+ n_jobs=1,
+ random_state=None,
+ verbose=0,
+ warm_start=False):
+ super(RotationForestRegressor, self).__init__(
+ base_estimator=RotationTreeRegressor(),
+ n_estimators=n_estimators,
+ estimator_params=("n_features_per_subset", "rotation_algo",
+ "criterion", "max_depth", "min_samples_split",
+ "min_samples_leaf", "min_weight_fraction_leaf",
+ "max_features", "max_leaf_nodes",
+ "random_state"),
+ bootstrap=bootstrap,
+ oob_score=oob_score,
+ n_jobs=n_jobs,
+ random_state=random_state,
+ verbose=verbose,
+ warm_start=warm_start)
+
+ self.n_features_per_subset = n_features_per_subset
+ self.rotation_algo = rotation_algo
+ self.criterion = criterion
+ self.max_depth = max_depth
+ self.min_samples_split = min_samples_split
+ self.min_samples_leaf = min_samples_leaf
+ self.min_weight_fraction_leaf = min_weight_fraction_leaf
+ self.max_features = max_features
+ self.max_leaf_nodes = max_leaf_nodes
diff --git a/rotation_forest/tests/test_rotation_forest.py b/rotation_forest/tests/test_rotation_forest.py
index a7b8c9d..e94e660 100644
--- a/rotation_forest/tests/test_rotation_forest.py
+++ b/rotation_forest/tests/test_rotation_forest.py
@@ -1,12 +1,12 @@
import pytest
-import copy
+#import copy
import numpy as np
import numpy.testing as npt
from sklearn.datasets import make_classification
-from sklearn.cross_validation import train_test_split
-from ..rotation_forest import random_feature_subsets
-from ..rotation_forest import RotationTreeClassifier, RotationForestClassifier
+from sklearn.model_selection import train_test_split
+from rotation_forest import random_feature_subsets
+from rotation_forest import RotationTreeClassifier, RotationForestClassifier
def classification_data(n_samples=500, n_features=30, redundant_size=0.1, informative_size=0.6, repeated_size=0.1):
@@ -65,7 +65,7 @@ def test_random_feature_subsets_batch_size_too_big(self):
array = np.arange(6*6).reshape(6, 6)
subsets = list(random_feature_subsets(array, batch_size=8))
assert len(subsets) == 1
- assert sorted(subsets[0]) == range(6)
+ assert sorted(subsets[0]) == list(range(6))
def test_rotation_matrix(self):
""" Smoke test for rotation forest """
diff --git a/setup.py b/setup.py
index 6239398..79364dc 100644
--- a/setup.py
+++ b/setup.py
@@ -9,10 +9,16 @@
def setup_package():
setup(
name="rotation_forest",
- version='0.1.0',
+ version='1.0',
description='Sklearn style implementation of the Rotation Forest Algorithm',
- author='Joshua D. Loyal',
- url='https://github.com/joshloyal/RotationForest',
+ long_description='''
+ Implementation of the Algorithm: J. J. Rodriguez, L. I. Kuncheva and C. J. Alonso, "Rotation Forest: A New Classifier Ensemble Method," in IEEE Transactions on Pattern Analysis and Machine Intelligence, vol. 28, no. 10, pp. 1619-1630, Oct. 2006.
+doi: 10.1109/TPAMI.2006.211''',
+ author='Joshua D. Loyal, Abhisek Maiti',
+ author_email='mail2abhisek.maiti@gmail.com',
+ maintainer='Abhisek Maiti',
+ maintainer_email='mail2abhisek.maiti@gmail.com',
+ url='https://github.com/digital-idiot/RotationForest',
license='MIT',
install_requires=['numpy', 'scipy', 'scikit-learn'],
packages=PACKAGES,