diff --git a/experiments/logistic_regression/decision_boundary.py b/experiments/logistic_regression/decision_boundary.py new file mode 100644 index 0000000..5d41765 --- /dev/null +++ b/experiments/logistic_regression/decision_boundary.py @@ -0,0 +1,50 @@ +import numpy as np +from glassboxml.data.generators import generate_classification_dataset +from glassboxml.models.logistic_regression import LogisticRegression +import matplotlib.pyplot as plt + +w_true = np.array([-2.5]) +b_true = 0.0 + +X, y, _, _ = generate_classification_dataset( + w_true, + b_true=b_true, + n_samples=10, + noise_std=0.1, + random_seed=42 +) + +print(X) +print(y) + +weights = [] + +model = LogisticRegression() +model.fit(X, y, epochs=1000, learning_rate=0.06) + +print(model.w, model.b) + +y_pred = model.predict(X) + +x_vals = np.linspace(min(X), max(X), 100) #Creates 100 evenly spaced values between the minimum and maximum of X for plotting the decision boundary +z = model.w[0] * x_vals + model.b # Computes the linear combination of inputs and weights for the decision boundary +y_probs = 1 / (1 + np.exp(-z)) # Applies the sigmoid function to the linear combination to get predicted probabilities for the decision boundary +x_boundary = -model.b / model.w[0] # Calculates the x-value where the decision boundary occurs (where predicted probability is 0.5) by setting the linear combination to zero and solving for x + +plt.figure() + +plt.scatter(X, y, label="Data") +plt.plot(x_vals, y_probs, color='red', label='Sigmoid') + +plt.axvline(x=x_boundary, color='green', linestyle='--', label='Boundary') + +y_pred_probs = model.predict_proba(X) + +plt.scatter(X, y_pred_probs, color='orange', label="Predicted probs") + +plt.xlabel("X") +plt.ylabel("Probability") +plt.legend() +plt.title("Logistic Regression Fit") + +plt.show() \ No newline at end of file diff --git a/experiments/logistic_regression/loss_convergence.py b/experiments/logistic_regression/loss_convergence.py new file mode 100644 index 0000000..3d51aca --- /dev/null +++ b/experiments/logistic_regression/loss_convergence.py @@ -0,0 +1,38 @@ +import numpy as np +from glassboxml.data.generators import generate_classification_dataset +from glassboxml.models.logistic_regression import LogisticRegression +import matplotlib.pyplot as plt + +w_true = np.array([2.5]) +b_true = 0.0 + +X, y, _, _ = generate_classification_dataset( + w_true, + b_true=b_true, + n_samples=10, + noise_std=0.1, + random_seed=42 +) + +print(X) +print(y) + +weights = [] + +model = LogisticRegression() +model.fit(X, y, epochs=1000, learning_rate=0.06) + +print(model.w, model.b) + +y_pred = model.predict(X) + +plt.figure() + +plt.plot(model.losses) +plt.yscale('log') +plt.xlabel('Epoch') +plt.ylabel('Loss') +plt.title('Loss Convergence') + +plt.grid(True) +plt.show() \ No newline at end of file diff --git a/glassboxml/data/generators.py b/glassboxml/data/generators.py index 98a7e95..8b2cd9d 100644 --- a/glassboxml/data/generators.py +++ b/glassboxml/data/generators.py @@ -43,4 +43,52 @@ def generate_regression_dataset( #Calculates true values of y: X multiplies true weights, @ is a matrix multiplication operator y = X @ w_true + b_true + noise + return X, y, w_true, b_true + +def generate_classification_dataset( + w_true, + b_true = 0.0, + n_samples=1000, + noise_std=0.01, + random_seed=42 +): + """ + Generate synthetic classification dataset. + + y = sigmoid(Xw + b + noise) + + Parameters + ---------- + w_true : array-like + Ground truth weight vector. + n_samples : int + Number of samples to generate. + noise_std : float + Standard deviation of Gaussian noise. + random_seed : int + For reproducibility. + + Returns + ------- + X : ndarray + y : ndarray + """ + + np.random.seed(random_seed) + + w_true = np.array(w_true) + n_features = len(w_true) + + #Generates input values with normal distribution where mean = 0, variance = 1 + X = np.random.randn(n_samples, n_features) + + #Generates one noise value per sample + noise = np.random.normal(0, noise_std, size=n_samples) + + #Calculates true values of y: X multiplies true weights, @ is a matrix multiplication operator + linear_output = X @ w_true + b_true + noise + + #Applies sigmoid function to get probabilities and then thresholds at 0.5 to get binary labels + y = (1 / (1 + np.exp(-linear_output)) >= 0.5).astype(int) + return X, y, w_true, b_true \ No newline at end of file diff --git a/glassboxml/models/logistic_regression.py b/glassboxml/models/logistic_regression.py new file mode 100644 index 0000000..c95e7e9 --- /dev/null +++ b/glassboxml/models/logistic_regression.py @@ -0,0 +1,55 @@ +import numpy as np +from glassboxml.losses.regularization import L1Regularization, L2Regularization +from glassboxml.optimizers.gradient_descent import GradientDescent + +class LogisticRegression: + def __init__(self, regularization=None): + self.regularization = regularization + self.epsilon = 1e-6 + self.losses = [] + pass + + def sigmoid(self, z): + return 1 / (1 + np.exp(-z)) + + def predict_proba(self, X): + z = X @ self.w + self.b + return self.sigmoid(z) + + def predict(self, X): + return (self.predict_proba(X) >= 0.5).astype(int) + + def fit(self, X, y, epochs, learning_rate): + n_samples = X.shape[0] + n_features = X.shape[1] + self.w = np.zeros(n_features) + self.b = 0 + optimizer = GradientDescent(learning_rate) + + for epoch in range(epochs): + y_pred = self.predict_proba(X) + + dw = 1/n_samples * X.T @ (y_pred- y) + (self.regularization.gradient(self.w) if self.regularization else 0) + db = 1/n_samples * np.sum(y_pred - y) + + if np.linalg.norm(dw) < self.epsilon and abs(db) < self.epsilon: + print("Converged at epoch", epoch) + break + + params ={ + 'w': self.w, + 'b': self.b + } + + grads = { + 'w': dw, + 'b': db + } + + optimizer.step(params, grads) + + loss = -np.mean(y * np.log(y_pred + self.epsilon) + (1 - y) * np.log(1 - y_pred + self.epsilon)) + self.losses.append(loss) + + self.w = params['w'] + self.b = params['b'] \ No newline at end of file diff --git a/tests/losses/test_regularization.py b/tests/losses/test_regularization.py index ea72e80..ba98c0e 100644 --- a/tests/losses/test_regularization.py +++ b/tests/losses/test_regularization.py @@ -1,5 +1,3 @@ -from glassboxml.data.generators import generate_regression_dataset -from glassboxml.models.linear_regression import LinearRegression from glassboxml.losses.regularization import L1Regularization, L2Regularization import numpy as np diff --git a/tests/models/test_logistic_regression.py b/tests/models/test_logistic_regression.py new file mode 100644 index 0000000..861aa37 --- /dev/null +++ b/tests/models/test_logistic_regression.py @@ -0,0 +1,44 @@ +import numpy as np +from glassboxml.data.generators import generate_classification_dataset +from glassboxml.models.logistic_regression import LogisticRegression + +def test_logistic_regression_direction_similarity(): + # Generate synthetic data + w_true = np.array([2.54321, 1.23456, 0.98765]) + b_true = 0.54321 + + X, y, _, _ = generate_classification_dataset( + w_true, + b_true=b_true, + n_samples=1000, + noise_std=0.1 + ) + + # Test direction similarity of learned weights to true weights + model = LogisticRegression() + model.fit(X, y, epochs=1000, learning_rate=0.01) + cos_sim = np.dot(model.w, w_true) / ( + np.linalg.norm(model.w) * np.linalg.norm(w_true) + ) + + assert cos_sim > 0.99 + +def test_logistic_regression_prediction_accuracy(): + # Generate synthetic data + w_true = np.array([2.54321, 1.23456, 0.98765]) + b_true = 0.54321 + + X, y, _, _ = generate_classification_dataset( + w_true, + b_true=b_true, + n_samples=1000, + noise_std=0.1 + ) + + # Test prediction accuracy of the model + model = LogisticRegression() + model.fit(X, y, epochs=1000, learning_rate=0.01) + y_pred = model.predict(X) + + accuracy = np.mean(y_pred == y) + assert accuracy > 0.95 \ No newline at end of file