From b6136d8302fcc36691eb4d0ecb5b1b65b71c80ed Mon Sep 17 00:00:00 2001 From: Iskken Date: Mon, 16 Mar 2026 20:40:13 +0100 Subject: [PATCH 1/6] Implement basic logistic regression --- glassboxml/models/logistic_regression.py | 51 ++++++++++++++++++++++++ 1 file changed, 51 insertions(+) create mode 100644 glassboxml/models/logistic_regression.py diff --git a/glassboxml/models/logistic_regression.py b/glassboxml/models/logistic_regression.py new file mode 100644 index 0000000..2d6fbcc --- /dev/null +++ b/glassboxml/models/logistic_regression.py @@ -0,0 +1,51 @@ +import numpy as np +from glassboxml.losses.regularization import L1Regularization, L2Regularization +from glassboxml.optimizers.gradient_descent import GradientDescent + +class LogisticRegression: + def __init__(self, regularization=None): + self.regularization = regularization + self.epsilon = 1e-6 + pass + + def sigmoid(self, z): + return 1 / (1 + np.exp(-z)) + + def predict_proba(self, X): + z = X @ self.w + self.b + return self.sigmoid(z) + + def predict(self, X): + return (self.predict_proba(X) >= 0.5).astype(int) + + def fit(self, X, y, epochs, learning_rate): + n_samples = X.shape[0] + n_features = X.shape[1] + self.w = np.zeros(n_features) + self.b = 0 + optimizer = GradientDescent(learning_rate) + + for epoch in range(epochs): + y_pred = self.predict_proba(X) + + dw = 1/n_samples * X.T @ (y_pred- y) + (self.regularization.gradient(self.w) if self.regularization else 0) + db = 1/n_samples * np.sum(y_pred - y) + + if np.linalg.norm(dw) < self.epsilon and abs(db) < self.epsilon: + print("Converged at epoch", epoch) + break + + params ={ + 'w': self.w, + 'b': self.b + } + + grads = { + 'w': dw, + 'b': db + } + + optimizer.step(params, grads) + + self.w = params['w'] + self.b = params['b'] \ No newline at end of file From 7cebafda7dfd8cfc66b23ad1c0471e708c0c6b30 Mon Sep 17 00:00:00 2001 From: Iskken Date: Mon, 16 Mar 2026 20:47:35 +0100 Subject: [PATCH 2/6] Implement classification dataset generation --- glassboxml/data/generators.py | 48 +++++++++++++++++++++++++++++++ tests/test_logistic_regression.py | 5 ++++ 2 files changed, 53 insertions(+) create mode 100644 tests/test_logistic_regression.py diff --git a/glassboxml/data/generators.py b/glassboxml/data/generators.py index 98a7e95..8b2cd9d 100644 --- a/glassboxml/data/generators.py +++ b/glassboxml/data/generators.py @@ -43,4 +43,52 @@ def generate_regression_dataset( #Calculates true values of y: X multiplies true weights, @ is a matrix multiplication operator y = X @ w_true + b_true + noise + return X, y, w_true, b_true + +def generate_classification_dataset( + w_true, + b_true = 0.0, + n_samples=1000, + noise_std=0.01, + random_seed=42 +): + """ + Generate synthetic classification dataset. + + y = sigmoid(Xw + b + noise) + + Parameters + ---------- + w_true : array-like + Ground truth weight vector. + n_samples : int + Number of samples to generate. + noise_std : float + Standard deviation of Gaussian noise. + random_seed : int + For reproducibility. + + Returns + ------- + X : ndarray + y : ndarray + """ + + np.random.seed(random_seed) + + w_true = np.array(w_true) + n_features = len(w_true) + + #Generates input values with normal distribution where mean = 0, variance = 1 + X = np.random.randn(n_samples, n_features) + + #Generates one noise value per sample + noise = np.random.normal(0, noise_std, size=n_samples) + + #Calculates true values of y: X multiplies true weights, @ is a matrix multiplication operator + linear_output = X @ w_true + b_true + noise + + #Applies sigmoid function to get probabilities and then thresholds at 0.5 to get binary labels + y = (1 / (1 + np.exp(-linear_output)) >= 0.5).astype(int) + return X, y, w_true, b_true \ No newline at end of file diff --git a/tests/test_logistic_regression.py b/tests/test_logistic_regression.py new file mode 100644 index 0000000..6a091e4 --- /dev/null +++ b/tests/test_logistic_regression.py @@ -0,0 +1,5 @@ +import numpy as np +from glassboxml.models.logistic_regression import LogisticRegression + +def test_logistic_regression_convergence(): + \ No newline at end of file From e4de9f7d91974e0449c5a6f18395e7cf4178d41a Mon Sep 17 00:00:00 2001 From: Iskken Date: Tue, 17 Mar 2026 14:58:47 +0100 Subject: [PATCH 3/6] Implement tests for logistic regression --- tests/test_logistic_regression.py | 43 +++++++++++++++++++++++++++++-- 1 file changed, 41 insertions(+), 2 deletions(-) diff --git a/tests/test_logistic_regression.py b/tests/test_logistic_regression.py index 6a091e4..861aa37 100644 --- a/tests/test_logistic_regression.py +++ b/tests/test_logistic_regression.py @@ -1,5 +1,44 @@ import numpy as np +from glassboxml.data.generators import generate_classification_dataset from glassboxml.models.logistic_regression import LogisticRegression -def test_logistic_regression_convergence(): - \ No newline at end of file +def test_logistic_regression_direction_similarity(): + # Generate synthetic data + w_true = np.array([2.54321, 1.23456, 0.98765]) + b_true = 0.54321 + + X, y, _, _ = generate_classification_dataset( + w_true, + b_true=b_true, + n_samples=1000, + noise_std=0.1 + ) + + # Test direction similarity of learned weights to true weights + model = LogisticRegression() + model.fit(X, y, epochs=1000, learning_rate=0.01) + cos_sim = np.dot(model.w, w_true) / ( + np.linalg.norm(model.w) * np.linalg.norm(w_true) + ) + + assert cos_sim > 0.99 + +def test_logistic_regression_prediction_accuracy(): + # Generate synthetic data + w_true = np.array([2.54321, 1.23456, 0.98765]) + b_true = 0.54321 + + X, y, _, _ = generate_classification_dataset( + w_true, + b_true=b_true, + n_samples=1000, + noise_std=0.1 + ) + + # Test prediction accuracy of the model + model = LogisticRegression() + model.fit(X, y, epochs=1000, learning_rate=0.01) + y_pred = model.predict(X) + + accuracy = np.mean(y_pred == y) + assert accuracy > 0.95 \ No newline at end of file From da795a24e1eb70ae56aa03a7392635608b14c0cf Mon Sep 17 00:00:00 2001 From: Iskken Date: Tue, 17 Mar 2026 16:50:15 +0100 Subject: [PATCH 4/6] Observe loss convergence of logistic regression --- .../logistic_regression/loss_convergence.py | 38 +++++++++++++++++++ glassboxml/models/logistic_regression.py | 4 ++ 2 files changed, 42 insertions(+) create mode 100644 experiments/logistic_regression/loss_convergence.py diff --git a/experiments/logistic_regression/loss_convergence.py b/experiments/logistic_regression/loss_convergence.py new file mode 100644 index 0000000..3d51aca --- /dev/null +++ b/experiments/logistic_regression/loss_convergence.py @@ -0,0 +1,38 @@ +import numpy as np +from glassboxml.data.generators import generate_classification_dataset +from glassboxml.models.logistic_regression import LogisticRegression +import matplotlib.pyplot as plt + +w_true = np.array([2.5]) +b_true = 0.0 + +X, y, _, _ = generate_classification_dataset( + w_true, + b_true=b_true, + n_samples=10, + noise_std=0.1, + random_seed=42 +) + +print(X) +print(y) + +weights = [] + +model = LogisticRegression() +model.fit(X, y, epochs=1000, learning_rate=0.06) + +print(model.w, model.b) + +y_pred = model.predict(X) + +plt.figure() + +plt.plot(model.losses) +plt.yscale('log') +plt.xlabel('Epoch') +plt.ylabel('Loss') +plt.title('Loss Convergence') + +plt.grid(True) +plt.show() \ No newline at end of file diff --git a/glassboxml/models/logistic_regression.py b/glassboxml/models/logistic_regression.py index 2d6fbcc..c95e7e9 100644 --- a/glassboxml/models/logistic_regression.py +++ b/glassboxml/models/logistic_regression.py @@ -6,6 +6,7 @@ class LogisticRegression: def __init__(self, regularization=None): self.regularization = regularization self.epsilon = 1e-6 + self.losses = [] pass def sigmoid(self, z): @@ -47,5 +48,8 @@ def fit(self, X, y, epochs, learning_rate): optimizer.step(params, grads) + loss = -np.mean(y * np.log(y_pred + self.epsilon) + (1 - y) * np.log(1 - y_pred + self.epsilon)) + self.losses.append(loss) + self.w = params['w'] self.b = params['b'] \ No newline at end of file From c8a6f1adb1a5f5467fa94f04bc6c5c9e81e89cfe Mon Sep 17 00:00:00 2001 From: Iskken Date: Tue, 17 Mar 2026 16:52:43 +0100 Subject: [PATCH 5/6] Delete unnecessary imports --- tests/losses/test_regularization.py | 2 -- tests/{ => models}/test_logistic_regression.py | 0 2 files changed, 2 deletions(-) rename tests/{ => models}/test_logistic_regression.py (100%) diff --git a/tests/losses/test_regularization.py b/tests/losses/test_regularization.py index ea72e80..ba98c0e 100644 --- a/tests/losses/test_regularization.py +++ b/tests/losses/test_regularization.py @@ -1,5 +1,3 @@ -from glassboxml.data.generators import generate_regression_dataset -from glassboxml.models.linear_regression import LinearRegression from glassboxml.losses.regularization import L1Regularization, L2Regularization import numpy as np diff --git a/tests/test_logistic_regression.py b/tests/models/test_logistic_regression.py similarity index 100% rename from tests/test_logistic_regression.py rename to tests/models/test_logistic_regression.py From a6339a66c8223000a9afb4dd9a9efba101dca5e4 Mon Sep 17 00:00:00 2001 From: Iskken Date: Wed, 18 Mar 2026 18:46:33 +0100 Subject: [PATCH 6/6] Plot decision boundary for logistic regression --- .../logistic_regression/decision_boundary.py | 50 +++++++++++++++++++ 1 file changed, 50 insertions(+) create mode 100644 experiments/logistic_regression/decision_boundary.py diff --git a/experiments/logistic_regression/decision_boundary.py b/experiments/logistic_regression/decision_boundary.py new file mode 100644 index 0000000..5d41765 --- /dev/null +++ b/experiments/logistic_regression/decision_boundary.py @@ -0,0 +1,50 @@ +import numpy as np +from glassboxml.data.generators import generate_classification_dataset +from glassboxml.models.logistic_regression import LogisticRegression +import matplotlib.pyplot as plt + +w_true = np.array([-2.5]) +b_true = 0.0 + +X, y, _, _ = generate_classification_dataset( + w_true, + b_true=b_true, + n_samples=10, + noise_std=0.1, + random_seed=42 +) + +print(X) +print(y) + +weights = [] + +model = LogisticRegression() +model.fit(X, y, epochs=1000, learning_rate=0.06) + +print(model.w, model.b) + +y_pred = model.predict(X) + +x_vals = np.linspace(min(X), max(X), 100) #Creates 100 evenly spaced values between the minimum and maximum of X for plotting the decision boundary +z = model.w[0] * x_vals + model.b # Computes the linear combination of inputs and weights for the decision boundary +y_probs = 1 / (1 + np.exp(-z)) # Applies the sigmoid function to the linear combination to get predicted probabilities for the decision boundary +x_boundary = -model.b / model.w[0] # Calculates the x-value where the decision boundary occurs (where predicted probability is 0.5) by setting the linear combination to zero and solving for x + +plt.figure() + +plt.scatter(X, y, label="Data") +plt.plot(x_vals, y_probs, color='red', label='Sigmoid') + +plt.axvline(x=x_boundary, color='green', linestyle='--', label='Boundary') + +y_pred_probs = model.predict_proba(X) + +plt.scatter(X, y_pred_probs, color='orange', label="Predicted probs") + +plt.xlabel("X") +plt.ylabel("Probability") +plt.legend() +plt.title("Logistic Regression Fit") + +plt.show() \ No newline at end of file