Fixed the issue with deprecated RandomizedPCA and an import error

.gitignore

@@ -15,3 +15,4 @@ rgforest/rgfpackage/test
 rgfpackage/bin/
 rgfpackage/test/
 htmlcov/
+.idea

.travis.yml

@@ -0,0 +1,17 @@
+language: python
+python:
+- '3.7'
+- '3.8'
+- '3.9'
+install:
+- pip install -r requirements.txt
+- pip install .
+script:
+- pytest -v
+deploy:
+  provider: pypi
+  username: __token__
+  password:
+    secure: qoKQp5jbF9iJs/vmTB//5mPrjOtcCc5zouED3Qo9Ws6RTl+WRTkdpCsufZDSflfjX1hgsOkySYtbE6A3DyyGPEyTnkc91sp5YPGCNApqiCl7Y7l5LKrmqqw99LXB+h+9luz2YzgDPY6oYiPbNhQuksWk8jj3QNWSBAwHRhj5Qul96BvBQYm/a0jXEJF+WRh2q+tSj6EwNrNHhrD7Rs8YMRkHnx1vsz5PnegvqajIfUp7QWx8XiRSBCR+x2cLU/QivtakAZITZ5cJ/WeX2XRmczFSmJZ0zC4KnytASwr56wnhwCOT0oNmh413E8IZ6a/TTffN4wcXZhZdRoAAGuy9fqKWXoPcuq+72kjwPTM6SMW1FvBt+Bo94y5rAh/eXC25M0TTqHsdPpJJa/14nA993xtIXAer4VOk3UNmc6ioLYkrOlvU85XkpO076Ll8NQfzseJMsCb7FZjQ5xy48TLGqOFYA9qBDnVBf2/BHAfjVbBL7XRjJCJOFCqhVmM48bxagoZhPDDzPc0uXhv8KGott2arAtlsgc91zStMHzM6COyJ5V3lIS/ccwY5QFjkbW8xzOwVlO0nHSNCYNmO4LFb29cNe/F645RNeGhNdMYED81sKG7Lh8dXlliqu5lslKRQBmNtV/gf4HdRnmbKldFYAWU0p2dJHj42kTRygVbvW7M=
+  on:
+    tags: true

README.md

@@ -1,22 +1,41 @@
 Rotation Forest
-===============
-Simple hack to sklearn's random forest module to implement the Rotation Forest algorithm from Rodriguez et al. 2006.
+---
+[![Build Status](https://travis-ci.org/digital-idiot/RotationForest.svg?branch=master)](https://travis-ci.org/digital-idiot/RotationForest) [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
+---
+Simple derived implementation of <b>Rotation Forest</b> algorithm<sup>[1, 2]</sup>  based upon sklearn's random forest module.
 
 
 Algorithm
----------
+---
+```
 for tree in trees:
-- split the attributes in the training set into K non-overlapping subsets of equal size.
-- bootstrap 75% of the data from each K dataset and use the bootstrap data in the following steps.
-- Run PCA on the ith subset in K. Retain all principal components. For every feature j in the Kth subsets, we have a principal component a.
+	split the attributes in the training set into K non-overlapping subsets of equal size.
+	bootstrap þ% of the data from each K dataset and use the bootstrap data in the following steps:
+		- Run PCA on the i-th subset in K. Retain all principal components. For every feature j in the Kth subsets, we have a principal component a.
+		- Create a rotation matrix of size n X n where n is the total number of features. Arrange the principal component in the matrix such that the components match the position of the feature in the original training dataset.
+		- Project the training dataset on the rotation matrix.
+		- Build a decision tree with the projected dataset
+		- Store the tree and the rotation matrix.
+```
 
-Create a rotation matrix of size n X n where n is the total number of features. Arrange
-the principal component in the matrix such that the components match the position of the feature in the original training dataset.
+* Rotation forest has been implemented both as classifier<sup>[1]</sup> and regressor<sup>[2]</sup>.
 
-Project the training dataset on the rotation matrix.
 
-Build a decision tree with the projected dataset
+<sup>
+[1] J. J. Rodriguez, L. I. Kuncheva, and C. J. Alonso, “Rotation Forest: A New Classifier Ensemble Method,” IEEE Trans. Pattern Anal. Mach. Intell., vol. 28, no. 10, pp. 1619–1630, Oct. 2006, doi: 10.1109/tpami.2006.211. 
+</sup>
 
-Store the tree and the rotation matrix.
+<sup>
+[2] C. Pardo, J. F. Diez-Pastor, C. García-Osorio, and J. J. Rodríguez, “Rotation Forests for regression,” Applied Mathematics and Computation, vol. 219, no. 19, pp. 9914–9924, Jun. 2013, doi: 10.1016/j.amc.2013.03.139. 
+</sup>
 
-![Toy data benchmark](benchmarks/simple_benchmark.png)
+---
+
+## Toy Benchmark
+* ### Random Forest Classifier
+
+![Toy data benchmark](benchmarks/simple_benchmark_classification.png)
+
+* ### Random Forest Regressor
+
+![Toy data benchmark](benchmarks/simple_benchmark_regression.png)

benchmarks/benchmarks.py

@@ -1,13 +1,13 @@
 #!/usr/bin/python
 
-import os
 import time
 import random
 import numpy as np
+from pathlib import Path
 
-from sklearn import cross_validation
+# from sklearn import cross_validation
 from sklearn.metrics import accuracy_score
-from sklearn.cross_validation import StratifiedKFold
+from sklearn.model_selection import StratifiedKFold
 
 from sklearn.ensemble import RandomForestClassifier
 from rotation_forest import RotationForestClassifier
@@ -52,28 +52,38 @@ def read_uci_dataset(path):
     return X, Y
 
 if __name__ == '__main__':
+    c_dir = Path('../')
+    dataset_dir = (c_dir / 'datasets') / 'classification_data'
+    print(c_dir.absolute())
+    for dataset in dataset_dir.glob('*.data'):
+        name = dataset.name.split('.')[0]
 
-    dataset_dir = './datasets/'
-    for dataset in os.listdir(dataset_dir):
-        name = dataset.split('.')[0]
+        X, Y = read_uci_dataset(dataset)
 
-        X, Y = read_uci_dataset(dataset_dir+dataset)
-
-        K = 10
+        k = 10
         accuracy = []
-        cv = StratifiedKFold(Y, K)
+        skf = StratifiedKFold(
+            n_splits=k, shuffle=True, random_state=1234
+        )
 
-        print name
+        print(name)
         for clf in [RotationForestClassifier(), RandomForestClassifier(n_estimators=10)]:
-            for train, test in cv:
+            #for train, test in cv:
+            for train_index, test_index in skf.split(X, Y):
 
-                x_train, x_test, y_train, y_test = X[train,:], X[test,:], Y[train], Y[test]
+                #x_train, x_test, y_train, y_test = X[train,:], X[test,:], Y[train], Y[test]
+                x_train, x_test = X[train_index], X[test_index]
+                y_train, y_test = Y[train_index], Y[test_index]
 
                 clf = clf.fit(x_train, y_train)
                 y_pred = clf.predict(x_test)
                 accuracy.append(accuracy_score(y_test, y_pred))
 
             bd_std = np.std(accuracy)
             bd_acc = np.mean(accuracy)
-            print '{0}: {1:2f} +/- {2:2f}'.format(clf.__class__.__name__, bd_acc*100, bd_std*100)
-        print ""
+            print(
+                '{0}: {1:2f} +/- {2:2f}'.format(
+                    clf.__class__.__name__, bd_acc*100, bd_std*100
+                )
+            )
+        print()

benchmarks/simple_benchmark.py → benchmarks/simple_benchmark_classification.py

@@ -5,7 +5,7 @@
 from sklearn.ensemble import RandomForestClassifier, BaggingClassifier
 from sklearn.ensemble import AdaBoostClassifier
 from sklearn.tree import DecisionTreeClassifier
-from sklearn.cross_validation import StratifiedKFold
+from sklearn.model_selection import StratifiedKFold
 from sklearn.metrics import roc_auc_score
 
 from sklearn.pipeline import make_pipeline
@@ -20,20 +20,23 @@ def classification_data():
     n_informative = int(0.6 * n_features)
     n_repeated = int(0.1 * n_features)
 
-    X, y = make_classification(n_samples=500, n_features=n_features, flip_y=0.03,
-                               n_informative=n_informative, n_redundant=n_redundant,
-                               n_repeated=n_repeated, random_state=9)
+    X, y = make_classification(
+        n_samples=500, n_features=n_features, flip_y=0.03,
+        n_informative=n_informative, n_redundant=n_redundant,
+        n_repeated=n_repeated, random_state=9
+    )
 
     return X, y
 
 def test_toy_data(name, clf):
     X, y = classification_data()
     k_folds = 5
-    cv = StratifiedKFold(y, k_folds, random_state=1234)
+    skf = StratifiedKFold(n_splits=k_folds, shuffle=True, random_state=1234)
 
     acc, auc = [], []
-    for train, test in cv:
-        xt, xv, yt, yv = X[train, :], X[test, :], y[train], y[test]
+    for train_index, test_index in skf.split(X, y):
+        xt, xv = X[train_index], X[test_index]
+        yt, yv = y[train_index], y[test_index]
         clf.fit(xt, yt)
         yhat = clf.predict(xv)
         proba = clf.predict_proba(xv)[:, 1]
@@ -42,10 +45,10 @@ def test_toy_data(name, clf):
 
     acc_mean, acc_std = np.mean(acc), np.std(acc)
     auc_mean, auc_std = np.mean(auc), np.std(auc)
-    print name
-    print 'accuracy: {0:.3f} +/- {1:.3f}'.format(acc_mean, acc_std)
-    print 'auc: {0:.3f} +/- {1:.3f}'.format(auc_mean, auc_std)
-    print '-'*80
+    print(name)
+    print('accuracy: {0:.3f} +/- {1:.3f}'.format(acc_mean, acc_std))
+    print('auc: {0:.3f} +/- {1:.3f}'.format(auc_mean, auc_std))
+    print('-'*80)
     return {'name': name,
             'acc_mean': acc_mean,
             'acc_std': acc_std,
@@ -55,30 +58,29 @@ def test_toy_data(name, clf):
 classifiers = [('Random Forest',
                RandomForestClassifier(random_state=12, n_estimators=25)),
               ('PCA + Random Forest',
-               make_pipeline(PCA(), RandomForestClassifier(random_state=12,
-                                                           n_estimators=25))),
+               make_pipeline(PCA(), RandomForestClassifier(
+                   random_state=12, n_estimators=25))),
               ('Rotation Tree',
                RotationTreeClassifier(random_state=12,
                                       n_features_per_subset=3)),
               ('Decision Tree',
                DecisionTreeClassifier(random_state=12)),
-              ('Rotation Forest (PCA)',
+              ('Rotation Forest\n(PCA)',
                RotationForestClassifier(random_state=12,
                                         n_estimators=25,
                                         n_features_per_subset=3)),
-              ('Rotation Forest (Randomized PCA)',
+              ('Rotation Forest\n(Randomized PCA)',
                RotationForestClassifier(random_state=12,
                                         n_estimators=25,
                                         n_features_per_subset=3,
                                         rotation_algo='randomized')),
-              ('Adaboost (Rotation Tree)',
-               AdaBoostClassifier(RotationTreeClassifier(n_features_per_subset=3,
-                                                         random_state=12,
-                                                         max_depth=3),
+              ('Adaboost\n(Rotation Tree)',
+               AdaBoostClassifier(RotationTreeClassifier(
+                   n_features_per_subset=3, random_state=12, max_depth=3),
                                   n_estimators=25,
                                   random_state=12)
               ),
-              ('Adaboost (Decision Tree)',
+              ('Adaboost\n(Decision Tree)',
                AdaBoostClassifier(DecisionTreeClassifier(random_state=12,
                                                          max_depth=3),
                                   n_estimators=25,
@@ -92,10 +94,35 @@ def test_toy_data(name, clf):
     for name, clf in classifiers:
         results.append(test_toy_data(name, clf))
     df = pd.DataFrame(results)
-    plt.figure(figsize=(15, 12))
-    sns.barplot(x='name', y='auc_mean', data=df.sort('auc_mean'),
-                xerr=df['auc_std'].values)
-    plt.xticks(rotation=45)
-    plt.xlabel('Classifier')
-    plt.ylabel('AUC')
-    plt.savefig('simple_benchmark.png')
+    df = df.sort_values('auc_mean')
+    fig = plt.figure(figsize=(15, 15))
+    n = len(df.index)
+    cm = plt.cm.get_cmap('rainbow', n)
+    bcolors = [cm(i) for i in range(n)]
+    ax = fig.add_subplot(111)
+    ax.bar(
+        x=(np.arange(n) - 0.15), 
+        height=df['auc_mean'],
+        bottom=0,
+        align='center',
+        color=bcolors,
+        yerr=df['auc_std'],
+        width=0.25
+    )
+    ax.bar(
+        x=(np.arange(n) + 0.15), 
+        height=df['acc_mean'],
+        bottom=0,
+        align='center',
+        color=bcolors,
+        yerr=df['acc_std'],
+        width=0.25
+    )
+    ax.set_title('Classification Benchmark')
+    ax.set_xlabel('Classifier')
+    ax.set_ylabel('AUC / Accuracy')
+    ax.set_xticks(np.arange(n))
+    ax.set_xticklabels(df['name'])
+    ax.set_ylim(bottom=0.5, top=1.0)
+    ax.tick_params(axis="x", rotation=45)
+    plt.savefig('simple_benchmark_classification.png')