Feature/logistic regression

experiments/logistic_regression/decision_boundary.py

@@ -0,0 +1,50 @@
+import numpy as np
+from glassboxml.data.generators import generate_classification_dataset
+from glassboxml.models.logistic_regression import LogisticRegression
+import matplotlib.pyplot as plt
+
+w_true = np.array([-2.5])
+b_true = 0.0
+
+X, y, _, _ = generate_classification_dataset(
+    w_true,
+    b_true=b_true,
+    n_samples=10,
+    noise_std=0.1,
+    random_seed=42
+)
+
+print(X)
+print(y)
+
+weights = []
+
+model = LogisticRegression()
+model.fit(X, y, epochs=1000, learning_rate=0.06)
+
+print(model.w, model.b)
+
+y_pred = model.predict(X)
+
+x_vals = np.linspace(min(X), max(X), 100) #Creates 100 evenly spaced values between the minimum and maximum of X for plotting the decision boundary
+z = model.w[0] * x_vals + model.b # Computes the linear combination of inputs and weights for the decision boundary
+y_probs = 1 / (1 + np.exp(-z)) # Applies the sigmoid function to the linear combination to get predicted probabilities for the decision boundary
+x_boundary = -model.b / model.w[0] # Calculates the x-value where the decision boundary occurs (where predicted probability is 0.5) by setting the linear combination to zero and solving for x
+
+plt.figure()
+
+plt.scatter(X, y, label="Data")
+plt.plot(x_vals, y_probs, color='red', label='Sigmoid')
+
+plt.axvline(x=x_boundary, color='green', linestyle='--', label='Boundary')
+
+y_pred_probs = model.predict_proba(X)
+
+plt.scatter(X, y_pred_probs, color='orange', label="Predicted probs")
+
+plt.xlabel("X")
+plt.ylabel("Probability")
+plt.legend()
+plt.title("Logistic Regression Fit")
+
+plt.show()

experiments/logistic_regression/loss_convergence.py

@@ -0,0 +1,38 @@
+import numpy as np
+from glassboxml.data.generators import generate_classification_dataset
+from glassboxml.models.logistic_regression import LogisticRegression
+import matplotlib.pyplot as plt
+
+w_true = np.array([2.5])
+b_true = 0.0
+
+X, y, _, _ = generate_classification_dataset(
+    w_true,
+    b_true=b_true,
+    n_samples=10,
+    noise_std=0.1,
+    random_seed=42
+)
+
+print(X)
+print(y)
+
+weights = []
+
+model = LogisticRegression()
+model.fit(X, y, epochs=1000, learning_rate=0.06)
+
+print(model.w, model.b)
+
+y_pred = model.predict(X)
+
+plt.figure()
+
+plt.plot(model.losses)
+plt.yscale('log')
+plt.xlabel('Epoch')
+plt.ylabel('Loss')
+plt.title('Loss Convergence')
+
+plt.grid(True)
+plt.show()

glassboxml/data/generators.py

@@ -43,4 +43,52 @@ def generate_regression_dataset(
     #Calculates true values of y: X multiplies true weights, @ is a matrix multiplication operator
     y = X @ w_true + b_true + noise
 
+    return X, y, w_true, b_true
+
+def generate_classification_dataset(
+    w_true,
+    b_true = 0.0,
+    n_samples=1000,
+    noise_std=0.01,
+    random_seed=42
+):
+    """
+    Generate synthetic classification dataset.
+
+    y = sigmoid(Xw + b + noise)
+
+    Parameters
+    ----------
+    w_true : array-like
+        Ground truth weight vector.
+    n_samples : int
+        Number of samples to generate.
+    noise_std : float
+        Standard deviation of Gaussian noise.
+    random_seed : int
+        For reproducibility.
+
+    Returns
+    -------
+    X : ndarray
+    y : ndarray
+    """
+
+    np.random.seed(random_seed)
+
+    w_true = np.array(w_true)
+    n_features = len(w_true)
+
+    #Generates input values with normal distribution where mean = 0, variance = 1
+    X = np.random.randn(n_samples, n_features)
+
+    #Generates one noise value per sample
+    noise = np.random.normal(0, noise_std, size=n_samples)  
+
+    #Calculates true values of y: X multiplies true weights, @ is a matrix multiplication operator
+    linear_output = X @ w_true + b_true + noise
+
+    #Applies sigmoid function to get probabilities and then thresholds at 0.5 to get binary labels
+    y = (1 / (1 + np.exp(-linear_output)) >= 0.5).astype(int)
+
     return X, y, w_true, b_true

glassboxml/models/logistic_regression.py

@@ -0,0 +1,55 @@
+import numpy as np
+from glassboxml.losses.regularization import L1Regularization, L2Regularization
+from glassboxml.optimizers.gradient_descent import GradientDescent
+
+class LogisticRegression:
+    def __init__(self, regularization=None):
+        self.regularization = regularization
+        self.epsilon = 1e-6
+        self.losses = []
+        pass
+
+    def sigmoid(self, z):
+        return 1 / (1 + np.exp(-z))
+
+    def predict_proba(self, X):
+        z = X @ self.w + self.b
+        return self.sigmoid(z)
+
+    def predict(self, X):
+        return (self.predict_proba(X) >= 0.5).astype(int)
+
+    def fit(self, X, y, epochs, learning_rate):
+        n_samples = X.shape[0]
+        n_features = X.shape[1]
+        self.w = np.zeros(n_features)
+        self.b = 0
+        optimizer = GradientDescent(learning_rate)
+
+        for epoch in range(epochs):
+            y_pred = self.predict_proba(X)
+
+            dw = 1/n_samples * X.T @ (y_pred- y) + (self.regularization.gradient(self.w) if self.regularization else 0)
+            db = 1/n_samples * np.sum(y_pred - y)
+
+            if np.linalg.norm(dw) < self.epsilon and abs(db) < self.epsilon:
+                print("Converged at epoch", epoch)
+                break
+
+            params ={
+                'w': self.w,
+                'b': self.b
+            }
+
+            grads = {
+                'w': dw,
+                'b': db
+            }
+
+            optimizer.step(params, grads)
+
+            loss = -np.mean(y * np.log(y_pred + self.epsilon) + (1 - y) * np.log(1 - y_pred + self.epsilon))
+            self.losses.append(loss)
+
+            self.w = params['w']
+            self.b = params['b']

tests/losses/test_regularization.py

@@ -1,5 +1,3 @@
-from glassboxml.data.generators import generate_regression_dataset
-from glassboxml.models.linear_regression import LinearRegression
 from glassboxml.losses.regularization import L1Regularization, L2Regularization
 import numpy as np
 

tests/models/test_logistic_regression.py

@@ -0,0 +1,44 @@
+import numpy as np
+from glassboxml.data.generators import generate_classification_dataset
+from glassboxml.models.logistic_regression import LogisticRegression
+
+def test_logistic_regression_direction_similarity():
+    # Generate synthetic data
+    w_true = np.array([2.54321, 1.23456, 0.98765])
+    b_true = 0.54321
+
+    X, y, _, _ = generate_classification_dataset(
+        w_true,
+        b_true=b_true,
+        n_samples=1000,
+        noise_std=0.1
+    )
+
+    # Test direction similarity of learned weights to true weights
+    model = LogisticRegression()
+    model.fit(X, y, epochs=1000, learning_rate=0.01)
+    cos_sim = np.dot(model.w, w_true) / (
+        np.linalg.norm(model.w) * np.linalg.norm(w_true)
+    )
+
+    assert cos_sim > 0.99
+
+def test_logistic_regression_prediction_accuracy():
+    # Generate synthetic data
+    w_true = np.array([2.54321, 1.23456, 0.98765])
+    b_true = 0.54321
+
+    X, y, _, _ = generate_classification_dataset(
+        w_true,
+        b_true=b_true,
+        n_samples=1000,
+        noise_std=0.1
+    )
+
+    # Test prediction accuracy of the model
+    model = LogisticRegression()
+    model.fit(X, y, epochs=1000, learning_rate=0.01)
+    y_pred = model.predict(X)
+
+    accuracy = np.mean(y_pred == y)
+    assert accuracy > 0.95