Feature/knn merge

.github/workflows/tests.yml

@@ -21,8 +21,7 @@ jobs:
 
       - name: Install dependencies
         run: |
-          pip install -e .
-          pip install pytest
+          pip install -e ".[test]"
 
       - name: Run tests
         run: pytest

.vscode/settings.json

@@ -0,0 +1,7 @@
+{
+    "python.testing.pytestArgs": [
+        "."
+    ],
+    "python.testing.unittestEnabled": false,
+    "python.testing.pytestEnabled": true
+}

Jenkinsfile

@@ -17,8 +17,7 @@ pipeline {
                 sh '''
                     python3 -m venv .venv
                     .venv/bin/pip install --upgrade pip
-                    .venv/bin/pip install -e .
-                    .venv/bin/pip install pytest
+                    .venv/bin/pip install -e ".[test]"
                 '''
             }
         }

README.md

@@ -25,6 +25,11 @@ Gini-impurity splitting, recursive binary partitioning, configurable max depth.
 
 ![Decision tree decision regions](docs/images/decision_tree.png)
 
+### K-Nearest Neighbors
+Instance-based classification (and regression) via majority vote / mean over the `k` closest training points, vectorized distance computation.
+
+![KNN decision regions](docs/images/knn.png)
+
 Plots were generated with `python docs/generate_plots.py` — rerun it after changing a model to refresh them.
 
 ## Project layout

docs/generate_plots.py

@@ -9,6 +9,7 @@
     generate_regression_dataset,
 )
 from glassboxml.models.decision_tree import DecisionTree
+from glassboxml.models.knn import KNNClassifier
 from glassboxml.models.linear_regression import LinearRegression
 from glassboxml.models.logistic_regression import LogisticRegression
 
@@ -87,8 +88,36 @@ def plot_decision_tree():
     plt.close()
 
 
+def plot_knn():
+    X, y, w_true, b_true = generate_classification_dataset(
+        w_true=[1.5, -2.0], b_true=0.5, n_samples=300, noise_std=0.5,
+    )
+    model = KNNClassifier(k=5)
+    model.fit(X, y)
+
+    x0_min, x0_max = X[:, 0].min() - 0.5, X[:, 0].max() + 0.5
+    x1_min, x1_max = X[:, 1].min() - 0.5, X[:, 1].max() + 0.5
+    xx0, xx1 = np.meshgrid(
+        np.linspace(x0_min, x0_max, 200),
+        np.linspace(x1_min, x1_max, 200),
+    )
+    grid = np.column_stack([xx0.ravel(), xx1.ravel()])
+    zz = model.predict(grid).reshape(xx0.shape)
+
+    plt.figure(figsize=(6, 4.5))
+    plt.contourf(xx0, xx1, zz, alpha=0.3, cmap="coolwarm")
+    plt.scatter(X[:, 0], X[:, 1], c=y, cmap="coolwarm", alpha=0.8, edgecolors="k", linewidths=0.3)
+    plt.title(f"K-Nearest Neighbors (k={model.k})")
+    plt.xlabel("x1")
+    plt.ylabel("x2")
+    plt.tight_layout()
+    plt.savefig(f"{OUT_DIR}/knn.png", dpi=150)
+    plt.close()
+
+
 if __name__ == "__main__":
     plot_linear_regression()
     plot_logistic_regression()
     plot_decision_tree()
+    plot_knn()
     print(f"Saved plots to {OUT_DIR}/")

experiments/knn/scratch.ipynb

@@ -0,0 +1,188 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "id": "8cb78c58-1868-4794-aef3-e8f6a68944c0",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from glassboxml.data.generators import generate_classification_dataset"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "id": "8eb1f929-9680-424e-83a5-eee52144719f",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "X, y, w_true, b_true = generate_classification_dataset([1.5, 2.0, 0.4, 0.1])"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "id": "330bc2ee",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from glassboxml.models.knn import KNN\n",
+    "from sklearn.model_selection import train_test_split\n",
+    "\n",
+    "X_train, X_test, y_train, y_test = train_test_split(\n",
+    "    X, y, test_size=0.33, random_state=42)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "id": "1b0358e9",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "model = KNN(5)\n",
+    "model.fit(X_train, y_train)\n",
+    "y_pred = model.predict(X_test)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "id": "b811db49",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "np.float64(0.9545454545454546)"
+      ]
+     },
+     "execution_count": 5,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "from glassboxml.metrics.classification import accuracy\n",
+    "\n",
+    "accuracy(y_test, y_pred)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "id": "b4a63140",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "predictions on training data: [0 0 0 1 1 1 2 2 2]\n",
+      "expected:                     [0 0 0 1 1 1 2 2 2]\n"
+     ]
+    }
+   ],
+   "source": [
+    "import numpy as np\n",
+    "X = np.array([[0.0],[0.1],[0.2], [5.0],[5.1],[5.2], [10.0],[10.1],[10.2]])\n",
+    "y = np.array([0,0,0, 1,1,1, 2,2,2])\n",
+    "\n",
+    "model = KNN(k=3)\n",
+    "model.fit(X, y)\n",
+    "print('predictions on training data:', model.predict(X))\n",
+    "print('expected:                    ', y)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "id": "9df85a4e",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "array([1.73205081, 6.4807407 , 3.74165739])"
+      ]
+     },
+     "execution_count": 7,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "X_sample = np.array([[1,1,1], [1,4,5], [3,2,1]])\n",
+    "x = np.array([0,0,0])\n",
+    "\n",
+    "np.linalg.norm(X_sample - x, axis=1)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "id": "ddf12c44",
+   "metadata": {},
+   "outputs": [
+    {
+     "ename": "AxisError",
+     "evalue": "axis 1 is out of bounds for array of dimension 1",
+     "output_type": "error",
+     "traceback": [
+      "\u001b[31m---------------------------------------------------------------------------\u001b[39m",
+      "\u001b[31mAxisError\u001b[39m                                 Traceback (most recent call last)",
+      "\u001b[36mCell\u001b[39m\u001b[36m \u001b[39m\u001b[32mIn[2]\u001b[39m\u001b[32m, line 10\u001b[39m\n\u001b[32m      7\u001b[39m model = KNNClassifier(k = \u001b[32m3\u001b[39m)\n\u001b[32m      8\u001b[39m model.fit(X,y)\n\u001b[32m---> \u001b[39m\u001b[32m10\u001b[39m y_pred = \u001b[43mmodel\u001b[49m\u001b[43m.\u001b[49m\u001b[43mpredict\u001b[49m\u001b[43m(\u001b[49m\u001b[43mX\u001b[49m\u001b[43m)\u001b[49m\n",
+      "\u001b[36mFile \u001b[39m\u001b[32m~/GlassBoxML/glassboxml/models/knn.py:13\u001b[39m, in \u001b[36m_KNNBase.predict\u001b[39m\u001b[34m(self, X)\u001b[39m\n\u001b[32m     12\u001b[39m \u001b[38;5;28;01mdef\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34mpredict\u001b[39m(\u001b[38;5;28mself\u001b[39m, X):\n\u001b[32m---> \u001b[39m\u001b[32m13\u001b[39m     \u001b[38;5;28;01mreturn\u001b[39;00m np.array(\u001b[43m[\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43m_predict_sample\u001b[49m\u001b[43m(\u001b[49m\u001b[43mx\u001b[49m\u001b[43m)\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43;01mfor\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[43mx\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;129;43;01min\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[43mnp\u001b[49m\u001b[43m.\u001b[49m\u001b[43marray\u001b[49m\u001b[43m(\u001b[49m\u001b[43mX\u001b[49m\u001b[43m)\u001b[49m\u001b[43m]\u001b[49m)\n",
+      "\u001b[36mFile \u001b[39m\u001b[32m~/GlassBoxML/glassboxml/models/knn.py:13\u001b[39m, in \u001b[36m<listcomp>\u001b[39m\u001b[34m(.0)\u001b[39m\n\u001b[32m     12\u001b[39m \u001b[38;5;28;01mdef\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34mpredict\u001b[39m(\u001b[38;5;28mself\u001b[39m, X):\n\u001b[32m---> \u001b[39m\u001b[32m13\u001b[39m     \u001b[38;5;28;01mreturn\u001b[39;00m np.array([\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43m_predict_sample\u001b[49m\u001b[43m(\u001b[49m\u001b[43mx\u001b[49m\u001b[43m)\u001b[49m \u001b[38;5;28;01mfor\u001b[39;00m x \u001b[38;5;129;01min\u001b[39;00m np.array(X)])\n",
+      "\u001b[36mFile \u001b[39m\u001b[32m~/GlassBoxML/glassboxml/models/knn.py:37\u001b[39m, in \u001b[36mKNNClassifier._predict_sample\u001b[39m\u001b[34m(self, x)\u001b[39m\n\u001b[32m     36\u001b[39m \u001b[38;5;28;01mdef\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34m_predict_sample\u001b[39m(\u001b[38;5;28mself\u001b[39m, x):\n\u001b[32m---> \u001b[39m\u001b[32m37\u001b[39m     k_labels = \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43m_k_nearest_labels\u001b[49m\u001b[43m(\u001b[49m\u001b[43mx\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m     38\u001b[39m     \u001b[38;5;28;01mreturn\u001b[39;00m Counter(k_labels).most_common(\u001b[32m1\u001b[39m)[\u001b[32m0\u001b[39m][\u001b[32m0\u001b[39m]\n",
+      "\u001b[36mFile \u001b[39m\u001b[32m~/GlassBoxML/glassboxml/models/knn.py:17\u001b[39m, in \u001b[36m_KNNBase._k_nearest_labels\u001b[39m\u001b[34m(self, x)\u001b[39m\n\u001b[32m     15\u001b[39m \u001b[38;5;28;01mdef\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34m_k_nearest_labels\u001b[39m(\u001b[38;5;28mself\u001b[39m, x):\n\u001b[32m     16\u001b[39m     \u001b[38;5;66;03m#Use vectorized calculations for calculating distances\u001b[39;00m\n\u001b[32m---> \u001b[39m\u001b[32m17\u001b[39m     distances = \u001b[43mnp\u001b[49m\u001b[43m.\u001b[49m\u001b[43mlinalg\u001b[49m\u001b[43m.\u001b[49m\u001b[43mnorm\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43mX_train\u001b[49m\u001b[43m \u001b[49m\u001b[43m-\u001b[49m\u001b[43m \u001b[49m\u001b[43mx\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43maxis\u001b[49m\u001b[43m=\u001b[49m\u001b[32;43m1\u001b[39;49m\u001b[43m)\u001b[49m\n\u001b[32m     19\u001b[39m     \u001b[38;5;66;03m# argpartition(distances, k) needs an extra element past the k we\u001b[39;00m\n\u001b[32m     20\u001b[39m     \u001b[38;5;66;03m# keep to anchor the partition, so it only works for k < n_samples.\u001b[39;00m\n\u001b[32m     21\u001b[39m     \u001b[38;5;66;03m# If k covers the whole training set (or more), there's nothing to\u001b[39;00m\n\u001b[32m     22\u001b[39m     \u001b[38;5;66;03m# partition - every point is a \"k nearest\" neighbour.\u001b[39;00m\n\u001b[32m     23\u001b[39m     \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m.k >= \u001b[38;5;28mlen\u001b[39m(distances):\n",
+      "\u001b[36mFile \u001b[39m\u001b[32m~/GlassBoxML/.venv/lib/python3.11/site-packages/numpy/linalg/_linalg.py:2804\u001b[39m, in \u001b[36mnorm\u001b[39m\u001b[34m(x, ord, axis, keepdims)\u001b[39m\n\u001b[32m   2801\u001b[39m \u001b[38;5;28;01melif\u001b[39;00m \u001b[38;5;28mord\u001b[39m \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mord\u001b[39m == \u001b[32m2\u001b[39m:\n\u001b[32m   2802\u001b[39m     \u001b[38;5;66;03m# special case for speedup\u001b[39;00m\n\u001b[32m   2803\u001b[39m     s = (x.conj() * x).real\n\u001b[32m-> \u001b[39m\u001b[32m2804\u001b[39m     \u001b[38;5;28;01mreturn\u001b[39;00m sqrt(\u001b[43madd\u001b[49m\u001b[43m.\u001b[49m\u001b[43mreduce\u001b[49m\u001b[43m(\u001b[49m\u001b[43ms\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43maxis\u001b[49m\u001b[43m=\u001b[49m\u001b[43maxis\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mkeepdims\u001b[49m\u001b[43m=\u001b[49m\u001b[43mkeepdims\u001b[49m\u001b[43m)\u001b[49m)\n\u001b[32m   2805\u001b[39m \u001b[38;5;66;03m# None of the str-type keywords for ord ('fro', 'nuc')\u001b[39;00m\n\u001b[32m   2806\u001b[39m \u001b[38;5;66;03m# are valid for vectors\u001b[39;00m\n\u001b[32m   2807\u001b[39m \u001b[38;5;28;01melif\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(\u001b[38;5;28mord\u001b[39m, \u001b[38;5;28mstr\u001b[39m):\n",
+      "\u001b[31mAxisError\u001b[39m: axis 1 is out of bounds for array of dimension 1"
+     ]
+    }
+   ],
+   "source": [
+    "from glassboxml.models.knn import KNNClassifier\n",
+    "import numpy as np\n",
+    "\n",
+    "X = np.array([0.0, 0.1, 0.2,  5.1, 5.2, 5.5,  10.245, 11, 12.5])\n",
+    "y = np.array([0, 0, 0,  1, 1, 1,  2, 2, 2])\n",
+    "\n",
+    "model = KNNClassifier(k = 3)\n",
+    "model.fit(X,y)\n",
+    "\n",
+    "y_pred = model.predict(X)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "4150c622",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": ".venv (3.11.7)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.7"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

glassboxml/distances/distances.py

@@ -0,0 +1,7 @@
+import numpy as np
+
+def calculate_euclidean(point1, point2):
+    if (len(point1) != len(point2)):
+        raise Exception("Dimensions of points are not equal!")
+
+    return np.sqrt(np.sum(np.square([point1[i] - point2[i] for i in range(len(point1))])))

glassboxml/models/knn.py

@@ -0,0 +1,43 @@
+import numpy as np
+from collections import Counter
+
+class _KNNBase:
+    def __init__(self, k=5):
+        self.k = k
+
+    def fit(self, X, y):
+        self.X_train = np.array(X)
+        self.y_train = np.array(y)
+
+    def predict(self, X):
+        return np.array([self._predict_sample(x) for x in np.array(X)])
+
+    def _k_nearest_labels(self, x):
+        #Use vectorized calculations for calculating distances
+        distances = np.linalg.norm(self.X_train - x, axis=1)
+
+        # argpartition(distances, k) needs an extra element past the k we
+        # keep to anchor the partition, so it only works for k < n_samples.
+        # If k covers the whole training set (or more), there's nothing to
+        # partition - every point is a "k nearest" neighbour.
+        if self.k >= len(distances):
+            return self.y_train
+
+        #Use argpartition to obtain k closest neighbours
+        partitioned_indices = np.argpartition(distances, self.k) #Put k closest elements at the front of the index list
+        k_closest_indices = partitioned_indices[:self.k]
+
+        return self.y_train[k_closest_indices]
+
+    def _predict_sample(self, x):
+        raise NotImplementedError
+
+class KNNClassifier(_KNNBase):
+    def _predict_sample(self, x):
+        k_labels = self._k_nearest_labels(x)
+        return Counter(k_labels).most_common(1)[0][0]
+
+class KNNRegressor(_KNNBase):
+    def _predict_sample(self, x):
+        k_labels = self._k_nearest_labels(x)
+        return np.mean(k_labels)

pyproject.toml

@@ -6,6 +6,9 @@ dependencies = [
     "matplotlib"
 ]
 
+[project.optional-dependencies]
+test = ["pytest", "scikit-learn"]
+
 [build-system]
 requires = ["setuptools"]
 build-backend = "setuptools.build_meta"