diff --git a/experiments/random_forest/scratch.ipynb b/experiments/random_forest/scratch.ipynb
new file mode 100644
index 0000000..c24bd12
--- /dev/null
+++ b/experiments/random_forest/scratch.ipynb
@@ -0,0 +1,245 @@
+{
+ "cells": [
+ {
+ "cell_type": "code",
+ "execution_count": 3,
+ "id": "97224de0",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "import numpy as np"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 36,
+ "id": "50446c72",
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "array([[2, 0, 0, 2],\n",
+ " [2, 0, 1, 1],\n",
+ " [3, 3, 2, 3]])"
+ ]
+ },
+ "execution_count": 36,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "X = [[1,2],[2,2],[3,1],[4,3]]\n",
+ "n = 3\n",
+ "\n",
+ "rng = np.random.default_rng()\n",
+ "\n",
+ "samples = np.array([rng.choice(4, size = len(X), replace=True) for i in range(n)])\n",
+ "\n",
+ "samples"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 44,
+ "id": "a436cbea",
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "[[[3, 1], [1, 2], [1, 2], [3, 1]],\n",
+ " [[3, 1], [1, 2], [2, 2], [2, 2]],\n",
+ " [[4, 3], [4, 3], [3, 1], [4, 3]]]"
+ ]
+ },
+ "execution_count": 44,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "[[X[i] for i in s] for s in samples]"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 41,
+ "id": "06238b06",
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "[[[3, 1], [1, 2], [1, 2], [3, 1]],\n",
+ " [[3, 1], [1, 2], [2, 2], [2, 2]],\n",
+ " [[4, 3], [4, 3], [3, 1], [4, 3]]]"
+ ]
+ },
+ "execution_count": 41,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "res = []\n",
+ "for s in samples:\n",
+ " res.append([X[i] for i in s])\n",
+ "\n",
+ "res"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 1,
+ "id": "7235fb9a",
+ "metadata": {},
+ "outputs": [
+ {
+ "ename": "NameError",
+ "evalue": "name 'np' is not defined",
+ "output_type": "error",
+ "traceback": [
+ "\u001b[31m---------------------------------------------------------------------------\u001b[39m",
+ "\u001b[31mNameError\u001b[39m Traceback (most recent call last)",
+ "\u001b[36mCell\u001b[39m\u001b[36m \u001b[39m\u001b[32mIn[1]\u001b[39m\u001b[32m, line 1\u001b[39m\n\u001b[32m----> \u001b[39m\u001b[32m1\u001b[39m ~\u001b[43mnp\u001b[49m.isin([[\u001b[32m0\u001b[39m],[\u001b[32m1\u001b[39m],[\u001b[32m2\u001b[39m],[\u001b[32m3\u001b[39m]], [[\u001b[32m0\u001b[39m],[\u001b[32m2\u001b[39m],[\u001b[32m2\u001b[39m],[\u001b[32m3\u001b[39m]])\n",
+ "\u001b[31mNameError\u001b[39m: name 'np' is not defined"
+ ]
+ }
+ ],
+ "source": [
+ "~np.isin([[0],[1],[2],[3]], [[0],[2],[2],[3]])"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 11,
+ "id": "0292374c",
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "array([0, 0, 1])"
+ ]
+ },
+ "execution_count": 11,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "from glassboxml.models.random_forest import RandomForest\n",
+ "\n",
+ "m = RandomForest(10)\n",
+ "\n",
+ "X = [[0], [1], [-1], [-2], [2], [3], [4], [5]]\n",
+ "y = [0, 0, 0, 0, 1, 1, 1, 1]\n",
+ "\n",
+ "m.fit(X,y)\n",
+ "\n",
+ "m.accuracy\n",
+ "\n",
+ "m.predict([[-5], [-3], [2]])"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 38,
+ "id": "66e886fb",
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "np.float64(0.5873015873015873)"
+ ]
+ },
+ "execution_count": 38,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "X = [[1,9], [0,8], [-1, 5], [-15,4],\n",
+ " [1,10], [1,19], [-150,150], [0,0],\n",
+ " [2,19], [12,15], [17,10], [10,-150],\n",
+ " [20,21], [29,28], [350,300], [1000,21]]\n",
+ "y = [0, 0, 0, 0,\n",
+ " 1,1,1,1,\n",
+ " 2,2,2,2,\n",
+ " 3,3,3,3]\n",
+ "\n",
+ "model = RandomForest(10, max_features=2)\n",
+ "model.fit(X, y)\n",
+ "\n",
+ "model.accuracy"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 13,
+ "id": "0126cb68",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "from glassboxml.models.decision_tree import DecisionTree\n",
+ "\n",
+ "m = DecisionTree(10)\n",
+ "\n",
+ "m.fit(X,y)\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 39,
+ "id": "d976ae48",
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "{0, 1, 2, 3}"
+ ]
+ },
+ "execution_count": 39,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "set([0,1,1,2,3])"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "9f00bded",
+ "metadata": {},
+ "outputs": [],
+ "source": []
+ }
+ ],
+ "metadata": {
+ "kernelspec": {
+ "display_name": ".venv (3.11.7.final.0)",
+ "language": "python",
+ "name": "python3"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 3
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython3",
+ "version": "3.11.7"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}
diff --git a/glassboxml/models/decision_tree.py b/glassboxml/models/decision_tree.py
index 7560daa..b6ba4ce 100644
--- a/glassboxml/models/decision_tree.py
+++ b/glassboxml/models/decision_tree.py
@@ -17,14 +17,20 @@ def __init__(self, best_feature, best_threshold, left_child, right_child):
self.right_child = right_child
class DecisionTree:
- def __init__(self, max_depth):
+ def __init__(self, max_depth, max_features=None):
self.max_depth = max_depth
+ # None = consider every feature at every split (current/unchanged behavior).
+ # Otherwise, the number of features to randomly sample per split (e.g. sqrt(n_features)).
+ self.max_features = max_features
self.root = None
def fit(self, X, y):
X = np.array(X)
y = np.array(y)
+ if self.max_features == None:
+ self.max_features = X.shape[1]
+
self.root = self.build_tree(X,y, 0)
def build_tree(self, X, y, depth):
@@ -73,6 +79,11 @@ def calculate_gini(self, y):
probs = counts / counts.sum()
return 1 - np.sum(probs ** 2)
+ def _sample_feature_indices(self, n_features):
+ rng = np.random.default_rng()
+
+ return rng.choice(n_features, size = self.max_features, replace=False)
+
def find_best_split(self, X, y):
n_samples, n_features = X.shape
best_IG = 0
@@ -83,7 +94,9 @@ def find_best_split(self, X, y):
gini_parent = self.calculate_gini(y)
- for f in range(n_features):
+ current_features = self._sample_feature_indices(n_features)
+
+ for f in current_features:
values = np.unique(X[:, f])
thresholds = (values[:-1] + values[1:])/2
for t in thresholds:
diff --git a/glassboxml/models/random_forest.py b/glassboxml/models/random_forest.py
new file mode 100644
index 0000000..4026886
--- /dev/null
+++ b/glassboxml/models/random_forest.py
@@ -0,0 +1,66 @@
+import numpy as np
+from concurrent.futures import ProcessPoolExecutor
+from itertools import repeat
+from scipy.stats import mode
+from glassboxml.models.decision_tree import DecisionTree
+from glassboxml.metrics.classification import accuracy
+
+def _fit_one_tree(train_X, train_y, X_oob, y_oob, max_features):
+ # Module-level (not a method) so it can be pickled and sent to a
+ # worker process - each tree only needs its own bootstrap sample and
+ # OOB set, so this can run fully independently of the other trees
+ model = DecisionTree(100, max_features)
+ model.fit(train_X, train_y)
+ tree_accuracy = accuracy(y_oob, model.predict(X_oob))
+ return model, tree_accuracy
+
+class RandomForest:
+ def __init__(self, n, max_features=None):
+ self.n = n
+ self.max_features = max_features
+
+ def fit(self, X, y):
+ self.X_train = np.array(X)
+ self.y_train = np.array(y)
+
+ if self.max_features == None:
+ self.max_features = np.int16(np.floor(np.sqrt(len(X[0]))))
+
+ #Bootstrapping
+ rng = np.random.default_rng()
+ # Create n samples for each tree where each sample is of size - len(X).
+ # Each point is selected with replacement.
+ indice_samples = rng.choice(len(X), size = (self.n, len(X)), replace=True)
+
+ # Create train dataset from sampled indices
+ train_X = self.X_train[indice_samples]
+ train_y = self.y_train[indice_samples]
+
+ #Out-of-Bag(OOB) test computations
+ orig_indices = np.arange(len(X))
+ test_OOB = [np.setdiff1d(orig_indices, s) for s in indice_samples]
+
+ # Training n trees on n samples - each tree is fit independently of
+ # the others, so hand the n fits out across worker processes instead
+ # of fitting them one at a time on a single core
+ X_oob = [self.X_train[oob] for oob in test_OOB]
+ y_oob = [self.y_train[oob] for oob in test_OOB]
+
+ with ProcessPoolExecutor() as executor:
+ results = list(executor.map(
+ _fit_one_tree, train_X, train_y, X_oob, y_oob, repeat(self.max_features, self.n)
+ ))
+
+ self.models = [model for model, _ in results]
+ self.accuracy = np.mean([tree_accuracy for _, tree_accuracy in results])
+
+ def predict(self, X):
+ # Each tree's own vectorized predict() returns every sample's
+ # prediction in one call - stacking these gives a (n_trees, n_samples)
+ # matrix where column j holds every tree's vote for sample j
+ t_pred = np.array([t.predict(X) for t in self.models])
+
+ # Majority vote per sample = the mode down each column (axis=0)
+ y_pred, _ = mode(t_pred, axis=0)
+
+ return y_pred
\ No newline at end of file
diff --git a/pyproject.toml b/pyproject.toml
index 65b0fde..7f80924 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -3,7 +3,8 @@ name = "glassboxml"
version = "0.1.0"
dependencies = [
"numpy",
- "matplotlib"
+ "matplotlib",
+ "scipy"
]
[project.optional-dependencies]
diff --git a/tests/models/test_random_forest.py b/tests/models/test_random_forest.py
new file mode 100644
index 0000000..493b05f
--- /dev/null
+++ b/tests/models/test_random_forest.py
@@ -0,0 +1,55 @@
+import numpy as np
+import pytest
+from glassboxml.models.random_forest import RandomForest
+from glassboxml.data.generators import generate_classification_dataset
+from sklearn.model_selection import train_test_split
+
+def test_pipeline_works():
+ X,y,_,_ = generate_classification_dataset(w_true=[1.5,0.1, 2.1], b_true=0.01, n_samples=1000)
+
+ model = RandomForest(10)
+ model.fit(X, y)
+
+ assert model.accuracy >= 0.9
+
+def test_output_shape_and_values():
+ X, y, _, _ = generate_classification_dataset(w_true=[1.5,0.1, 2.1], b_true=0.01, n_samples=1000)
+ X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.33, random_state=42)
+
+ model = RandomForest(10)
+ model.fit(X_train, y_train)
+
+ y_pred = model.predict(X_test)
+
+ assert y_pred.shape == (len(y_test),)
+
+ assert np.isin(y_pred, y_train, assume_unique=False).all()
+
+def test_all_trees_fit():
+ X, y, _, _ = generate_classification_dataset(w_true=[1.5,0.1, 2.1], b_true=0.01, n_samples=1000)
+ X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.33, random_state=42)
+
+ model = RandomForest(10)
+ model.fit(X_train, y_train)
+
+ assert len(model.models) == 10
+
+def test_default_max_features():
+ X, y, _, _ = generate_classification_dataset(w_true=[1.5,0.1, 2.1], b_true=0.01, n_samples=1000)
+ X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.33, random_state=42)
+
+ model = RandomForest(10)
+ model.fit(X_train, y_train)
+
+ n_features = len(X[0])
+
+ assert model.models[0].max_features == np.int16(np.floor(np.sqrt(n_features)))
+
+def test_max_features_larger_orig_features():
+ X, y, _, _ = generate_classification_dataset(w_true=[1.5,0.1, 2.1], b_true=0.01, n_samples=1000)
+ X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.33, random_state=42)
+
+ model = RandomForest(10, max_features=4)
+
+ with pytest.raises(ValueError):
+ model.fit(X_train, y_train)
\ No newline at end of file
diff --git a/webapp/app.py b/webapp/app.py
index 9d2eda7..febd53a 100644
--- a/webapp/app.py
+++ b/webapp/app.py
@@ -16,6 +16,7 @@
from glassboxml.models.linear_regression import LinearRegression
from glassboxml.models.logistic_regression import LogisticRegression
from glassboxml.models.knn import KNNRegressor, KNNClassifier
+from glassboxml.models.random_forest import RandomForest
STATIC = Path(__file__).parent / "static"
@@ -31,6 +32,8 @@ class RunRequest(BaseModel):
learning_rate: float = 0.1
max_depth: int = 3
k: int = 5
+ n_trees: int = 10
+ max_features: int = None
@app.get("/")
@@ -48,6 +51,8 @@ def run(req: RunRequest):
return _decision_tree(req)
if req.algorithm == "knn":
return _knn(req)
+ if req.algorithm == "random_forest":
+ return _random_forest(req)
return {"error": f"Unknown algorithm: {req.algorithm}"}
@@ -159,4 +164,36 @@ def _knn(req: RunRequest):
"accuracy": f"{round(accuracy * 100, 1)}%",
"k": req.k,
},
+ }
+
+def _random_forest(req: RunRequest):
+ X, y, _, _ = generate_classification_dataset(
+ w_true=[1.5, -2.0], b_true=0.5,
+ n_samples=req.n_samples, noise_std=req.noise_std,
+ )
+
+ model = RandomForest(n=req.n_trees, max_features=req.max_features)
+ model.fit(X,y)
+
+ y_pred = model.predict(X)
+ # Training accuracy: how well the forest fits the data it was shown -
+ # can look deceptively high since trees can nearly memorize training points
+ training_accuracy = float(accuracy_metric(y, y_pred))
+ # OOB accuracy: each tree scored on the points it never saw during its own
+ # training - a much more honest, generalization-like estimate, for free
+ oob_accuracy = float(model.accuracy)
+
+ return {
+ "scatter": {
+ "x1": X[:, 0].tolist(),
+ "x2": X[:, 1].tolist(),
+ "labels": y.tolist(),
+ "predicted": y_pred.tolist(),
+ },
+ "metrics": {
+ "training_accuracy": f"{round(training_accuracy * 100, 1)}%",
+ "oob_accuracy": f"{round(oob_accuracy * 100, 1)}%",
+ "n_trees": req.n_trees,
+ "max_features": int(model.max_features)
+ },
}
\ No newline at end of file
diff --git a/webapp/static/index.html b/webapp/static/index.html
index d1acfcd..63d9edf 100644
--- a/webapp/static/index.html
+++ b/webapp/static/index.html
@@ -96,6 +96,7 @@ GlassBoxML — Interactive Demo
+
@@ -134,6 +135,11 @@ GlassBoxML — Interactive Demo
{ id: "noise_std", label: "Noise std", value: 0.3, min: 0, max: 3, step: 0.1 },
{ id: "k", label: "k (neighbors)", value: 5, min: 1, max: 25, step: 1 },
],
+ random_forest: [
+ { id: "n_samples", label: "Samples", value: 200, min: 20, max: 500, step: 10 },
+ { id: "noise_std", label: "Noise std", value: 0.3, min: 0, max: 3, step: 0.1 },
+ { id: "n_trees", label: "Trees", value: 10, min: 1, max: 50, step: 1 },
+ ],
};
let chart = null;