From 3e0b3e962ab7398615cb8adc82644223d6aa0b4c Mon Sep 17 00:00:00 2001 From: Iskken Date: Fri, 17 Jul 2026 04:12:52 +0200 Subject: [PATCH 1/5] Optimize Random Forest --- experiments/random_forest/scratch.ipynb | 181 ++++++++++++++++++++++++ glassboxml/models/random_forest.py | 101 +++++++++++++ pyproject.toml | 3 +- 3 files changed, 284 insertions(+), 1 deletion(-) create mode 100644 experiments/random_forest/scratch.ipynb create mode 100644 glassboxml/models/random_forest.py diff --git a/experiments/random_forest/scratch.ipynb b/experiments/random_forest/scratch.ipynb new file mode 100644 index 0000000..bdbcffa --- /dev/null +++ b/experiments/random_forest/scratch.ipynb @@ -0,0 +1,181 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 3, + "id": "97224de0", + "metadata": {}, + "outputs": [], + "source": [ + "import numpy as np" + ] + }, + { + "cell_type": "code", + "execution_count": 36, + "id": "50446c72", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "array([[2, 0, 0, 2],\n", + " [2, 0, 1, 1],\n", + " [3, 3, 2, 3]])" + ] + }, + "execution_count": 36, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "X = [[1,2],[2,2],[3,1],[4,3]]\n", + "n = 3\n", + "\n", + "rng = np.random.default_rng()\n", + "\n", + "samples = np.array([rng.choice(4, size = len(X), replace=True) for i in range(n)])\n", + "\n", + "samples" + ] + }, + { + "cell_type": "code", + "execution_count": 44, + "id": "a436cbea", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[[[3, 1], [1, 2], [1, 2], [3, 1]],\n", + " [[3, 1], [1, 2], [2, 2], [2, 2]],\n", + " [[4, 3], [4, 3], [3, 1], [4, 3]]]" + ] + }, + "execution_count": 44, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "[[X[i] for i in s] for s in samples]" + ] + }, + { + "cell_type": "code", + "execution_count": 41, + "id": "06238b06", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[[[3, 1], [1, 2], [1, 2], [3, 1]],\n", + " [[3, 1], [1, 2], [2, 2], [2, 2]],\n", + " [[4, 3], [4, 3], [3, 1], [4, 3]]]" + ] + }, + "execution_count": 41, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "res = []\n", + "for s in samples:\n", + " res.append([X[i] for i in s])\n", + "\n", + "res" + ] + }, + { + "cell_type": "code", + "execution_count": 43, + "id": "7235fb9a", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "array([[False],\n", + " [ True],\n", + " [False],\n", + " [False]])" + ] + }, + "execution_count": 43, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "~np.isin([[0],[1],[2],[3]], [[0],[2],[2],[3]])" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "0d257ef9", + "metadata": {}, + "outputs": [ + { + "ename": "ValueError", + "evalue": "setting an array element with a sequence. The requested array has an inhomogeneous shape after 1 dimensions. The detected shape was (10,) + inhomogeneous part.", + "output_type": "error", + "traceback": [ + "\u001b[31m---------------------------------------------------------------------------\u001b[39m", + "\u001b[31mValueError\u001b[39m Traceback (most recent call last)", + "\u001b[36mCell\u001b[39m\u001b[36m \u001b[39m\u001b[32mIn[1]\u001b[39m\u001b[32m, line 8\u001b[39m\n\u001b[32m 5\u001b[39m X = [[\u001b[32m0\u001b[39m], [\u001b[32m1\u001b[39m], [-\u001b[32m1\u001b[39m], [-\u001b[32m2\u001b[39m], [\u001b[32m2\u001b[39m], [\u001b[32m3\u001b[39m], [\u001b[32m4\u001b[39m], [\u001b[32m5\u001b[39m]]\n\u001b[32m 6\u001b[39m y = [\u001b[32m0\u001b[39m, \u001b[32m0\u001b[39m, \u001b[32m0\u001b[39m, \u001b[32m0\u001b[39m, \u001b[32m1\u001b[39m, \u001b[32m1\u001b[39m, \u001b[32m1\u001b[39m, \u001b[32m1\u001b[39m]\n\u001b[32m----> \u001b[39m\u001b[32m8\u001b[39m \u001b[43mm\u001b[49m\u001b[43m.\u001b[49m\u001b[43mfit\u001b[49m\u001b[43m(\u001b[49m\u001b[43mX\u001b[49m\u001b[43m,\u001b[49m\u001b[43my\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 10\u001b[39m m.accuracy\n\u001b[32m 12\u001b[39m \u001b[38;5;66;03m# m.predict([[-5], [-3], [2]])\u001b[39;00m\n", + "\u001b[36mFile \u001b[39m\u001b[32m~/GlassBoxML/glassboxml/models/random_forest.py:66\u001b[39m, in \u001b[36mRandomForest.fit\u001b[39m\u001b[34m(self, X, y)\u001b[39m\n\u001b[32m 58\u001b[39m test_OOB = [np.setdiff1d(orig_indices, s) \u001b[38;5;28;01mfor\u001b[39;00m s \u001b[38;5;129;01min\u001b[39;00m indice_samples]\n\u001b[32m 60\u001b[39m \u001b[38;5;66;03m# Training n trees on n samples - each tree is fit independently of\u001b[39;00m\n\u001b[32m 61\u001b[39m \u001b[38;5;66;03m# the others, so hand the n fits out across worker processes instead\u001b[39;00m\n\u001b[32m 62\u001b[39m \u001b[38;5;66;03m# of fitting them one at a time on a single core\u001b[39;00m\n\u001b[32m 63\u001b[39m \u001b[38;5;66;03m# X_oob = [self.X_train[oob] for oob in test_OOB]\u001b[39;00m\n\u001b[32m 64\u001b[39m \u001b[38;5;66;03m# y_oob = [self.y_train[oob] for oob in test_OOB]\u001b[39;00m\n\u001b[32m---> \u001b[39m\u001b[32m66\u001b[39m X_oob = \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43mX_train\u001b[49m\u001b[43m[\u001b[49m\u001b[43mtest_OOB\u001b[49m\u001b[43m]\u001b[49m\n\u001b[32m 67\u001b[39m y_oob = \u001b[38;5;28mself\u001b[39m.y_train[test_OOB]\n\u001b[32m 69\u001b[39m \u001b[38;5;28;01mwith\u001b[39;00m ProcessPoolExecutor() \u001b[38;5;28;01mas\u001b[39;00m executor:\n", + "\u001b[31mValueError\u001b[39m: setting an array element with a sequence. The requested array has an inhomogeneous shape after 1 dimensions. The detected shape was (10,) + inhomogeneous part." + ] + } + ], + "source": [ + "from glassboxml.models.random_forest import RandomForest\n", + "\n", + "m = RandomForest(10)\n", + "\n", + "X = [[0], [1], [-1], [-2], [2], [3], [4], [5]]\n", + "y = [0, 0, 0, 0, 1, 1, 1, 1]\n", + "\n", + "m.fit(X,y)\n", + "\n", + "m.accuracy\n", + "\n", + "# m.predict([[-5], [-3], [2]])\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "0292374c", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": ".venv (3.11.7.final.0)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.7" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/glassboxml/models/random_forest.py b/glassboxml/models/random_forest.py new file mode 100644 index 0000000..ae2c11f --- /dev/null +++ b/glassboxml/models/random_forest.py @@ -0,0 +1,101 @@ +import numpy as np +from concurrent.futures import ProcessPoolExecutor +from scipy.stats import mode +from glassboxml.models.decision_tree import DecisionTree +from glassboxml.metrics.classification import accuracy + +def _fit_one_tree(train_X, train_y, X_oob, y_oob): + # Module-level (not a method) so it can be pickled and sent to a + # worker process - each tree only needs its own bootstrap sample and + # OOB set, so this can run fully independently of the other trees + model = DecisionTree(100) + model.fit(train_X, train_y) + tree_accuracy = accuracy(y_oob, model.predict(X_oob)) + return model, tree_accuracy + +class RandomForest: + def __init__(self, n): + self.n = n + + def fit(self, X, y): + self.X_train = np.array(X) + self.y_train = np.array(y) + + # #Bootstrapping + # rng = np.random.default_rng() + # indice_samples = np.array([rng.choice(len(X), size = len(X), replace=True) for i in range(self.n)]) + # train_X = [[self.X_train[i] for i in s] for s in indice_samples] + # train_y = [[self.y_train[i] for i in s] for s in indice_samples] + # # For each tree, the OOB indices are the training points that were + # # never picked in that tree's bootstrap sample (set difference) + # test_OOB = [np.setdiff1d(np.arange(len(X)), s) for s in indice_samples] + + # #Training N trees on N samples + # accuracies = [] + # self.models = [] + # for i in range(self.n): + # model = DecisionTree(100) + # model.fit(train_X[i], train_y[i]) + # y_pred = model.predict([self.X_train[j] for j in test_OOB[i]]) + # y_true = [self.y_train[j] for j in test_OOB[i]] + # accuracies.append(accuracy(y_pred, y_true)) + # self.models.append(model) + + # self.accuracy = np.mean(accuracies) + + + #Bootstrapping + rng = np.random.default_rng() + # Create n samples for each tree where each sample is of size - len(X). + # Each point is selected with replacement. + indice_samples = rng.choice(len(X), size = (self.n, len(X)), replace=True) + + # Create train dataset from sampled indices + train_X = self.X_train[indice_samples] + train_y = self.y_train[indice_samples] + + #Out-of-Bag(OOB) test computations + orig_indices = np.arange(len(X)) + test_OOB = [np.setdiff1d(orig_indices, s) for s in indice_samples] + + # Training n trees on n samples - each tree is fit independently of + # the others, so hand the n fits out across worker processes instead + # of fitting them one at a time on a single core + X_oob = [self.X_train[oob] for oob in test_OOB] + y_oob = [self.y_train[oob] for oob in test_OOB] + + with ProcessPoolExecutor() as executor: + results = list(executor.map(_fit_one_tree, train_X, train_y, X_oob, y_oob)) + + self.models = [model for model, _ in results] + self.accuracy = np.mean([tree_accuracy for _, tree_accuracy in results]) + + + + + def predict(self, X): + # p_pred = [] + # for x in X: + # p = [] + # for t in self.models: + # p.append(t.predict_sample(x, t.root)) + + # p_pred.append(p) + + # y_pred = [] + # for p in p_pred: + # from collections import Counter + # res = Counter(p).most_common(1)[0][0] + # y_pred.append(res) + + # return y_pred + + # Each tree's own vectorized predict() returns every sample's + # prediction in one call - stacking these gives a (n_trees, n_samples) + # matrix where column j holds every tree's vote for sample j + t_pred = np.array([t.predict(X) for t in self.models]) + + # Majority vote per sample = the mode down each column (axis=0) + y_pred, _ = mode(t_pred, axis=0) + + return y_pred \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index 65b0fde..7f80924 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,7 +3,8 @@ name = "glassboxml" version = "0.1.0" dependencies = [ "numpy", - "matplotlib" + "matplotlib", + "scipy" ] [project.optional-dependencies] From 07108bedf2d08336c5cb569ec956b4ff29c07de5 Mon Sep 17 00:00:00 2001 From: Iskken Date: Sat, 18 Jul 2026 00:56:59 +0200 Subject: [PATCH 2/5] Random Forest Fix: Make each split sample random features and use them --- experiments/random_forest/scratch.ipynb | 48 ++++++++++----------- glassboxml/models/decision_tree.py | 14 ++++++- glassboxml/models/random_forest.py | 55 +++++-------------------- 3 files changed, 44 insertions(+), 73 deletions(-) diff --git a/experiments/random_forest/scratch.ipynb b/experiments/random_forest/scratch.ipynb index bdbcffa..7a1dd43 100644 --- a/experiments/random_forest/scratch.ipynb +++ b/experiments/random_forest/scratch.ipynb @@ -92,22 +92,20 @@ }, { "cell_type": "code", - "execution_count": 43, + "execution_count": 1, "id": "7235fb9a", "metadata": {}, "outputs": [ { - "data": { - "text/plain": [ - "array([[False],\n", - " [ True],\n", - " [False],\n", - " [False]])" - ] - }, - "execution_count": 43, - "metadata": {}, - "output_type": "execute_result" + "ename": "NameError", + "evalue": "name 'np' is not defined", + "output_type": "error", + "traceback": [ + "\u001b[31m---------------------------------------------------------------------------\u001b[39m", + "\u001b[31mNameError\u001b[39m Traceback (most recent call last)", + "\u001b[36mCell\u001b[39m\u001b[36m \u001b[39m\u001b[32mIn[1]\u001b[39m\u001b[32m, line 1\u001b[39m\n\u001b[32m----> \u001b[39m\u001b[32m1\u001b[39m ~\u001b[43mnp\u001b[49m.isin([[\u001b[32m0\u001b[39m],[\u001b[32m1\u001b[39m],[\u001b[32m2\u001b[39m],[\u001b[32m3\u001b[39m]], [[\u001b[32m0\u001b[39m],[\u001b[32m2\u001b[39m],[\u001b[32m2\u001b[39m],[\u001b[32m3\u001b[39m]])\n", + "\u001b[31mNameError\u001b[39m: name 'np' is not defined" + ] } ], "source": [ @@ -116,21 +114,19 @@ }, { "cell_type": "code", - "execution_count": 1, - "id": "0d257ef9", + "execution_count": 3, + "id": "0292374c", "metadata": {}, "outputs": [ { - "ename": "ValueError", - "evalue": "setting an array element with a sequence. The requested array has an inhomogeneous shape after 1 dimensions. The detected shape was (10,) + inhomogeneous part.", - "output_type": "error", - "traceback": [ - "\u001b[31m---------------------------------------------------------------------------\u001b[39m", - "\u001b[31mValueError\u001b[39m Traceback (most recent call last)", - "\u001b[36mCell\u001b[39m\u001b[36m \u001b[39m\u001b[32mIn[1]\u001b[39m\u001b[32m, line 8\u001b[39m\n\u001b[32m 5\u001b[39m X = [[\u001b[32m0\u001b[39m], [\u001b[32m1\u001b[39m], [-\u001b[32m1\u001b[39m], [-\u001b[32m2\u001b[39m], [\u001b[32m2\u001b[39m], [\u001b[32m3\u001b[39m], [\u001b[32m4\u001b[39m], [\u001b[32m5\u001b[39m]]\n\u001b[32m 6\u001b[39m y = [\u001b[32m0\u001b[39m, \u001b[32m0\u001b[39m, \u001b[32m0\u001b[39m, \u001b[32m0\u001b[39m, \u001b[32m1\u001b[39m, \u001b[32m1\u001b[39m, \u001b[32m1\u001b[39m, \u001b[32m1\u001b[39m]\n\u001b[32m----> \u001b[39m\u001b[32m8\u001b[39m \u001b[43mm\u001b[49m\u001b[43m.\u001b[49m\u001b[43mfit\u001b[49m\u001b[43m(\u001b[49m\u001b[43mX\u001b[49m\u001b[43m,\u001b[49m\u001b[43my\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 10\u001b[39m m.accuracy\n\u001b[32m 12\u001b[39m \u001b[38;5;66;03m# m.predict([[-5], [-3], [2]])\u001b[39;00m\n", - "\u001b[36mFile \u001b[39m\u001b[32m~/GlassBoxML/glassboxml/models/random_forest.py:66\u001b[39m, in \u001b[36mRandomForest.fit\u001b[39m\u001b[34m(self, X, y)\u001b[39m\n\u001b[32m 58\u001b[39m test_OOB = [np.setdiff1d(orig_indices, s) \u001b[38;5;28;01mfor\u001b[39;00m s \u001b[38;5;129;01min\u001b[39;00m indice_samples]\n\u001b[32m 60\u001b[39m \u001b[38;5;66;03m# Training n trees on n samples - each tree is fit independently of\u001b[39;00m\n\u001b[32m 61\u001b[39m \u001b[38;5;66;03m# the others, so hand the n fits out across worker processes instead\u001b[39;00m\n\u001b[32m 62\u001b[39m \u001b[38;5;66;03m# of fitting them one at a time on a single core\u001b[39;00m\n\u001b[32m 63\u001b[39m \u001b[38;5;66;03m# X_oob = [self.X_train[oob] for oob in test_OOB]\u001b[39;00m\n\u001b[32m 64\u001b[39m \u001b[38;5;66;03m# y_oob = [self.y_train[oob] for oob in test_OOB]\u001b[39;00m\n\u001b[32m---> \u001b[39m\u001b[32m66\u001b[39m X_oob = \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43mX_train\u001b[49m\u001b[43m[\u001b[49m\u001b[43mtest_OOB\u001b[49m\u001b[43m]\u001b[49m\n\u001b[32m 67\u001b[39m y_oob = \u001b[38;5;28mself\u001b[39m.y_train[test_OOB]\n\u001b[32m 69\u001b[39m \u001b[38;5;28;01mwith\u001b[39;00m ProcessPoolExecutor() \u001b[38;5;28;01mas\u001b[39;00m executor:\n", - "\u001b[31mValueError\u001b[39m: setting an array element with a sequence. The requested array has an inhomogeneous shape after 1 dimensions. The detected shape was (10,) + inhomogeneous part." - ] + "data": { + "text/plain": [ + "array([0, 0, 1])" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" } ], "source": [ @@ -145,13 +141,13 @@ "\n", "m.accuracy\n", "\n", - "# m.predict([[-5], [-3], [2]])\n" + "m.predict([[-5], [-3], [2]])" ] }, { "cell_type": "code", "execution_count": null, - "id": "0292374c", + "id": "66e886fb", "metadata": {}, "outputs": [], "source": [] diff --git a/glassboxml/models/decision_tree.py b/glassboxml/models/decision_tree.py index 7560daa..8ab6b41 100644 --- a/glassboxml/models/decision_tree.py +++ b/glassboxml/models/decision_tree.py @@ -17,8 +17,11 @@ def __init__(self, best_feature, best_threshold, left_child, right_child): self.right_child = right_child class DecisionTree: - def __init__(self, max_depth): + def __init__(self, max_depth, max_features=None): self.max_depth = max_depth + # None = consider every feature at every split (current/unchanged behavior). + # Otherwise, the number of features to randomly sample per split (e.g. sqrt(n_features)). + self.max_features = max_features self.root = None def fit(self, X, y): @@ -73,6 +76,11 @@ def calculate_gini(self, y): probs = counts / counts.sum() return 1 - np.sum(probs ** 2) + def _sample_feature_indices(self, n_features): + rng = np.random.default_rng() + + return rng.choice(n_features, size = self.max_features, replace=False) + def find_best_split(self, X, y): n_samples, n_features = X.shape best_IG = 0 @@ -83,7 +91,9 @@ def find_best_split(self, X, y): gini_parent = self.calculate_gini(y) - for f in range(n_features): + current_features = self._sample_feature_indices(n_features) + + for f in current_features: values = np.unique(X[:, f]) thresholds = (values[:-1] + values[1:])/2 for t in thresholds: diff --git a/glassboxml/models/random_forest.py b/glassboxml/models/random_forest.py index ae2c11f..4026886 100644 --- a/glassboxml/models/random_forest.py +++ b/glassboxml/models/random_forest.py @@ -1,48 +1,30 @@ import numpy as np from concurrent.futures import ProcessPoolExecutor +from itertools import repeat from scipy.stats import mode from glassboxml.models.decision_tree import DecisionTree from glassboxml.metrics.classification import accuracy -def _fit_one_tree(train_X, train_y, X_oob, y_oob): +def _fit_one_tree(train_X, train_y, X_oob, y_oob, max_features): # Module-level (not a method) so it can be pickled and sent to a # worker process - each tree only needs its own bootstrap sample and # OOB set, so this can run fully independently of the other trees - model = DecisionTree(100) + model = DecisionTree(100, max_features) model.fit(train_X, train_y) tree_accuracy = accuracy(y_oob, model.predict(X_oob)) return model, tree_accuracy class RandomForest: - def __init__(self, n): + def __init__(self, n, max_features=None): self.n = n + self.max_features = max_features def fit(self, X, y): self.X_train = np.array(X) self.y_train = np.array(y) - # #Bootstrapping - # rng = np.random.default_rng() - # indice_samples = np.array([rng.choice(len(X), size = len(X), replace=True) for i in range(self.n)]) - # train_X = [[self.X_train[i] for i in s] for s in indice_samples] - # train_y = [[self.y_train[i] for i in s] for s in indice_samples] - # # For each tree, the OOB indices are the training points that were - # # never picked in that tree's bootstrap sample (set difference) - # test_OOB = [np.setdiff1d(np.arange(len(X)), s) for s in indice_samples] - - # #Training N trees on N samples - # accuracies = [] - # self.models = [] - # for i in range(self.n): - # model = DecisionTree(100) - # model.fit(train_X[i], train_y[i]) - # y_pred = model.predict([self.X_train[j] for j in test_OOB[i]]) - # y_true = [self.y_train[j] for j in test_OOB[i]] - # accuracies.append(accuracy(y_pred, y_true)) - # self.models.append(model) - - # self.accuracy = np.mean(accuracies) - + if self.max_features == None: + self.max_features = np.int16(np.floor(np.sqrt(len(X[0])))) #Bootstrapping rng = np.random.default_rng() @@ -65,31 +47,14 @@ def fit(self, X, y): y_oob = [self.y_train[oob] for oob in test_OOB] with ProcessPoolExecutor() as executor: - results = list(executor.map(_fit_one_tree, train_X, train_y, X_oob, y_oob)) + results = list(executor.map( + _fit_one_tree, train_X, train_y, X_oob, y_oob, repeat(self.max_features, self.n) + )) self.models = [model for model, _ in results] self.accuracy = np.mean([tree_accuracy for _, tree_accuracy in results]) - - - def predict(self, X): - # p_pred = [] - # for x in X: - # p = [] - # for t in self.models: - # p.append(t.predict_sample(x, t.root)) - - # p_pred.append(p) - - # y_pred = [] - # for p in p_pred: - # from collections import Counter - # res = Counter(p).most_common(1)[0][0] - # y_pred.append(res) - - # return y_pred - # Each tree's own vectorized predict() returns every sample's # prediction in one call - stacking these gives a (n_trees, n_samples) # matrix where column j holds every tree's vote for sample j From 7010f2cc2eafb85bc97656cffe3e0bac12ff9e36 Mon Sep 17 00:00:00 2001 From: Iskken Date: Sat, 18 Jul 2026 01:11:17 +0200 Subject: [PATCH 3/5] Random Forest Fix: Fix the accuracy bug - returns Nan for no samples in OOB --- glassboxml/models/random_forest.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/glassboxml/models/random_forest.py b/glassboxml/models/random_forest.py index 4026886..f057705 100644 --- a/glassboxml/models/random_forest.py +++ b/glassboxml/models/random_forest.py @@ -52,7 +52,12 @@ def fit(self, X, y): )) self.models = [model for model, _ in results] - self.accuracy = np.mean([tree_accuracy for _, tree_accuracy in results]) + # A tree's bootstrap sample can, by chance, cover every original index, + # leaving its OOB set empty - accuracy() on an empty array is nan, and + # a plain mean would let a single nan wipe out the whole average. + # nanmean instead treats "no OOB samples for this tree" as no + # information, and just excludes it from the estimate. + self.accuracy = np.nanmean([tree_accuracy for _, tree_accuracy in results]) def predict(self, X): # Each tree's own vectorized predict() returns every sample's From ddda2bb4c30d309b8e89bbd7275c041688426ef9 Mon Sep 17 00:00:00 2001 From: Iskken Date: Sat, 18 Jul 2026 03:08:49 +0200 Subject: [PATCH 4/5] Random Forest Update: Implement tests for random forest --- experiments/random_forest/scratch.ipynb | 74 ++++++++++++++++++++++++- glassboxml/models/decision_tree.py | 3 + glassboxml/models/random_forest.py | 7 +-- tests/models/test_random_forest.py | 55 ++++++++++++++++++ 4 files changed, 130 insertions(+), 9 deletions(-) create mode 100644 tests/models/test_random_forest.py diff --git a/experiments/random_forest/scratch.ipynb b/experiments/random_forest/scratch.ipynb index 7a1dd43..c24bd12 100644 --- a/experiments/random_forest/scratch.ipynb +++ b/experiments/random_forest/scratch.ipynb @@ -114,7 +114,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 11, "id": "0292374c", "metadata": {}, "outputs": [ @@ -124,7 +124,7 @@ "array([0, 0, 1])" ] }, - "execution_count": 3, + "execution_count": 11, "metadata": {}, "output_type": "execute_result" } @@ -146,9 +146,77 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 38, "id": "66e886fb", "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "np.float64(0.5873015873015873)" + ] + }, + "execution_count": 38, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "X = [[1,9], [0,8], [-1, 5], [-15,4],\n", + " [1,10], [1,19], [-150,150], [0,0],\n", + " [2,19], [12,15], [17,10], [10,-150],\n", + " [20,21], [29,28], [350,300], [1000,21]]\n", + "y = [0, 0, 0, 0,\n", + " 1,1,1,1,\n", + " 2,2,2,2,\n", + " 3,3,3,3]\n", + "\n", + "model = RandomForest(10, max_features=2)\n", + "model.fit(X, y)\n", + "\n", + "model.accuracy" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "0126cb68", + "metadata": {}, + "outputs": [], + "source": [ + "from glassboxml.models.decision_tree import DecisionTree\n", + "\n", + "m = DecisionTree(10)\n", + "\n", + "m.fit(X,y)\n" + ] + }, + { + "cell_type": "code", + "execution_count": 39, + "id": "d976ae48", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{0, 1, 2, 3}" + ] + }, + "execution_count": 39, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "set([0,1,1,2,3])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9f00bded", + "metadata": {}, "outputs": [], "source": [] } diff --git a/glassboxml/models/decision_tree.py b/glassboxml/models/decision_tree.py index 8ab6b41..b6ba4ce 100644 --- a/glassboxml/models/decision_tree.py +++ b/glassboxml/models/decision_tree.py @@ -28,6 +28,9 @@ def fit(self, X, y): X = np.array(X) y = np.array(y) + if self.max_features == None: + self.max_features = X.shape[1] + self.root = self.build_tree(X,y, 0) def build_tree(self, X, y, depth): diff --git a/glassboxml/models/random_forest.py b/glassboxml/models/random_forest.py index f057705..4026886 100644 --- a/glassboxml/models/random_forest.py +++ b/glassboxml/models/random_forest.py @@ -52,12 +52,7 @@ def fit(self, X, y): )) self.models = [model for model, _ in results] - # A tree's bootstrap sample can, by chance, cover every original index, - # leaving its OOB set empty - accuracy() on an empty array is nan, and - # a plain mean would let a single nan wipe out the whole average. - # nanmean instead treats "no OOB samples for this tree" as no - # information, and just excludes it from the estimate. - self.accuracy = np.nanmean([tree_accuracy for _, tree_accuracy in results]) + self.accuracy = np.mean([tree_accuracy for _, tree_accuracy in results]) def predict(self, X): # Each tree's own vectorized predict() returns every sample's diff --git a/tests/models/test_random_forest.py b/tests/models/test_random_forest.py new file mode 100644 index 0000000..493b05f --- /dev/null +++ b/tests/models/test_random_forest.py @@ -0,0 +1,55 @@ +import numpy as np +import pytest +from glassboxml.models.random_forest import RandomForest +from glassboxml.data.generators import generate_classification_dataset +from sklearn.model_selection import train_test_split + +def test_pipeline_works(): + X,y,_,_ = generate_classification_dataset(w_true=[1.5,0.1, 2.1], b_true=0.01, n_samples=1000) + + model = RandomForest(10) + model.fit(X, y) + + assert model.accuracy >= 0.9 + +def test_output_shape_and_values(): + X, y, _, _ = generate_classification_dataset(w_true=[1.5,0.1, 2.1], b_true=0.01, n_samples=1000) + X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.33, random_state=42) + + model = RandomForest(10) + model.fit(X_train, y_train) + + y_pred = model.predict(X_test) + + assert y_pred.shape == (len(y_test),) + + assert np.isin(y_pred, y_train, assume_unique=False).all() + +def test_all_trees_fit(): + X, y, _, _ = generate_classification_dataset(w_true=[1.5,0.1, 2.1], b_true=0.01, n_samples=1000) + X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.33, random_state=42) + + model = RandomForest(10) + model.fit(X_train, y_train) + + assert len(model.models) == 10 + +def test_default_max_features(): + X, y, _, _ = generate_classification_dataset(w_true=[1.5,0.1, 2.1], b_true=0.01, n_samples=1000) + X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.33, random_state=42) + + model = RandomForest(10) + model.fit(X_train, y_train) + + n_features = len(X[0]) + + assert model.models[0].max_features == np.int16(np.floor(np.sqrt(n_features))) + +def test_max_features_larger_orig_features(): + X, y, _, _ = generate_classification_dataset(w_true=[1.5,0.1, 2.1], b_true=0.01, n_samples=1000) + X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.33, random_state=42) + + model = RandomForest(10, max_features=4) + + with pytest.raises(ValueError): + model.fit(X_train, y_train) \ No newline at end of file From 643082728bf33173995c0e24218ee0e02386b696 Mon Sep 17 00:00:00 2001 From: Iskken Date: Sat, 18 Jul 2026 03:41:09 +0200 Subject: [PATCH 5/5] Random Forest Update: Wire into the web app --- webapp/app.py | 37 +++++++++++++++++++++++++++++++++++++ webapp/static/index.html | 6 ++++++ 2 files changed, 43 insertions(+) diff --git a/webapp/app.py b/webapp/app.py index 9d2eda7..febd53a 100644 --- a/webapp/app.py +++ b/webapp/app.py @@ -16,6 +16,7 @@ from glassboxml.models.linear_regression import LinearRegression from glassboxml.models.logistic_regression import LogisticRegression from glassboxml.models.knn import KNNRegressor, KNNClassifier +from glassboxml.models.random_forest import RandomForest STATIC = Path(__file__).parent / "static" @@ -31,6 +32,8 @@ class RunRequest(BaseModel): learning_rate: float = 0.1 max_depth: int = 3 k: int = 5 + n_trees: int = 10 + max_features: int = None @app.get("/") @@ -48,6 +51,8 @@ def run(req: RunRequest): return _decision_tree(req) if req.algorithm == "knn": return _knn(req) + if req.algorithm == "random_forest": + return _random_forest(req) return {"error": f"Unknown algorithm: {req.algorithm}"} @@ -159,4 +164,36 @@ def _knn(req: RunRequest): "accuracy": f"{round(accuracy * 100, 1)}%", "k": req.k, }, + } + +def _random_forest(req: RunRequest): + X, y, _, _ = generate_classification_dataset( + w_true=[1.5, -2.0], b_true=0.5, + n_samples=req.n_samples, noise_std=req.noise_std, + ) + + model = RandomForest(n=req.n_trees, max_features=req.max_features) + model.fit(X,y) + + y_pred = model.predict(X) + # Training accuracy: how well the forest fits the data it was shown - + # can look deceptively high since trees can nearly memorize training points + training_accuracy = float(accuracy_metric(y, y_pred)) + # OOB accuracy: each tree scored on the points it never saw during its own + # training - a much more honest, generalization-like estimate, for free + oob_accuracy = float(model.accuracy) + + return { + "scatter": { + "x1": X[:, 0].tolist(), + "x2": X[:, 1].tolist(), + "labels": y.tolist(), + "predicted": y_pred.tolist(), + }, + "metrics": { + "training_accuracy": f"{round(training_accuracy * 100, 1)}%", + "oob_accuracy": f"{round(oob_accuracy * 100, 1)}%", + "n_trees": req.n_trees, + "max_features": int(model.max_features) + }, } \ No newline at end of file diff --git a/webapp/static/index.html b/webapp/static/index.html index d1acfcd..63d9edf 100644 --- a/webapp/static/index.html +++ b/webapp/static/index.html @@ -96,6 +96,7 @@

GlassBoxML — Interactive Demo

+
@@ -134,6 +135,11 @@

GlassBoxML — Interactive Demo

{ id: "noise_std", label: "Noise std", value: 0.3, min: 0, max: 3, step: 0.1 }, { id: "k", label: "k (neighbors)", value: 5, min: 1, max: 25, step: 1 }, ], + random_forest: [ + { id: "n_samples", label: "Samples", value: 200, min: 20, max: 500, step: 10 }, + { id: "noise_std", label: "Noise std", value: 0.3, min: 0, max: 3, step: 0.1 }, + { id: "n_trees", label: "Trees", value: 10, min: 1, max: 50, step: 1 }, + ], }; let chart = null;