diff --git a/Neural-Networks-From-Scratch-Connor.zip b/Neural-Networks-From-Scratch-Connor.zip new file mode 100644 index 0000000..6278333 Binary files /dev/null and b/Neural-Networks-From-Scratch-Connor.zip differ diff --git a/__pycache__/test.cpython-313.pyc b/__pycache__/test.cpython-313.pyc new file mode 100644 index 0000000..87e02df Binary files /dev/null and b/__pycache__/test.cpython-313.pyc differ diff --git a/back.py b/back.py new file mode 100644 index 0000000..6c4df3f --- /dev/null +++ b/back.py @@ -0,0 +1,51 @@ +x = [1.0, -2.0, 3.0] +w = [-3.0, -1.0, 2.0] +b = 1.0 + +xw0 = x[0] * w[0] +xw1 = x[1] * w[1] +xw2 = x[2] * w[2] + +z = xw0 + xw1 + xw2 + b +y = max(z, 0) + +dvalue = 1.0 +drelu_dz = dvalue * (1.0 if z > 0 else 0.0) + +dsum_dxw0 = 1 +drelu_dxw0 = drelu_dz * dsum_dxw0 + +dsum_dxw1 = 1 +drelu_dxw1 = drelu_dz * dsum_dxw1 + +dsum_dxw2 = 1 +drelu_dxw2 = drelu_dz * dsum_dxw2 + +dsum_db = 1 +drelu_db = drelu_dz * dsum_db + +# f = xy +# df/dx = y +# df/dy = x + +dmul_dx0 = w[0] +drelu_dx0 = drelu_dxw0 * dmul_dx0 + +dmul_dx1 = w[1] +drelu_dx1 = drelu_dxw1 * dmul_dx1 + +dmul_dx2 = w[2] +drelu_dx2 = drelu_dxw2 * dmul_dx2 + +dmul_dw0 = x[0] +drelu_dw0 = drelu_dxw0 * dmul_dw0 + +dmul_dw1 = x[1] +drelu_dw1 = drelu_dxw1 * dmul_dw1 + +dmul_dw2 = x[2] +drelu_dw2 = drelu_dxw2 * dmul_dw2 + +print(drelu_dx0, drelu_dx1, drelu_dx2) +print(drelu_dw0, drelu_dw1, drelu_dw2) +print(drelu_db) \ No newline at end of file diff --git a/ex.py b/ex.py new file mode 100644 index 0000000..677ee09 --- /dev/null +++ b/ex.py @@ -0,0 +1,60 @@ +import math +import numpy as np + +# output = [0.95, 0, 0] +target = [1, 0, 0] + +# # loss = -1 * (math.log(output[0]) * target[0] + +# # math.log(output[1]) * target[1] + +# # math.log(output[2]) * target[2]) + + +# loss = -1 * (math.log(output[0])) +# # loss = -log(output[0]) +# print(loss) + + + +# output = np.array([[0.7, 0.1, 0.2], +# [0.1, 0.5, 0.4], +# [0.02, 0.9, 0.08]]) + + +# #[0, 0, 1] +# print(-np.log(1.000000001)) + + +# targets = np.array([[1, 0, 0], +# [0, 1, 0], +# [0, 1, 0]]) +# if len(targets.shape) == 1: +# cc = output[ +# range(len(output)), +# targets +# ] +# elif len(targets.shape) == 2: +# cc = np.sum( +# output * targets, +# axis=1 +# ) +# avg = np.mean(-np.log(cc)) +# print(avg) +# preds = np.argmax(outputs, axis = 1) + +# accuracy = np.mean(predicitions) + + + + +outputs = np.array([[0.7, 0.1, 0.1], + [0.1, 0.5, 0.4], + [0.02, 0.9, 0.08]]) + +targets = np.array([0, 1, 1]) + +preds = np.argmax(outputs, axis = 1) +if len(targets.shape) == 2: + targets = np.argmax(targets, axis=1) + +accuracy = np.mean(preds == targets) +print(accuracy) \ No newline at end of file diff --git a/ex2.py b/ex2.py new file mode 100644 index 0000000..1fa8a7a --- /dev/null +++ b/ex2.py @@ -0,0 +1,53 @@ +import matplotlib.pyplot as plt + +import nnfs +from nnfs.datasets import vertical_data +from test import * +nnfs.init() + +X, y = vertical_data(samples = 100, classes = 3) +# plt.scatter(X[:, 0], X[:, 1], c=y, s=40, cmap='brg') +# plt.show() + +dense1 = layer_dense(2, 3) +activation1 = activate_ReLU() +dense2 = layer_dense(3, 3) +activation2 = activate_Softmax() + +loss_fn = Categorical_CrossEntropy() + +lowest_loss = 9999999 +best_dense1_weights = dense1.weights.copy() +best_dense1_biases = dense1.biases.copy() + +best_dense2_weights = dense2.weights.copy() +best_dense2_biases = dense2.biases.copy() + +for i in range(10000): + dense1.weights += 0.05 * np.random.randn(2, 3) + dense1.biases += 0.05 * np.random.randn(1, 3) + dense2.weights += 0.05 * np.random.randn(3, 3) + dense2.biases += 0.05 * np.random.randn(1, 3) + + dense1.forward(X) + activation1.forward(dense1.output) + dense2.forward(activation1.output) + activation2.forward(dense2.output) + + loss = loss_fn.calculate(activation2.output, y) + + predictions = np.argmax(activation2.output, axis=1) + accuracy = np.mean(predictions == y) + + if loss < lowest_loss: + print(f"New set of weights found, iteration {i}, loss: {loss}, accuracy: {accuracy}") + best_dense1_weights = dense1.weights.copy() + best_dense1_biases = dense1.biases.copy() + best_dense2_weights = dense2.weights.copy() + best_dense2_biases = dense2.biases.copy() + lowest_loss = loss + else: + dense1.weights = best_dense1_weights.copy() + dense1.biases = best_dense1_biases.copy() + dense2.weights = best_dense2_weights.copy() + dense2.biases = best_dense2_biases.copy() \ No newline at end of file diff --git a/ex3.py b/ex3.py new file mode 100644 index 0000000..9cc76a1 --- /dev/null +++ b/ex3.py @@ -0,0 +1,4 @@ +import numpy as np + +x = np.eye(5)[1] +print(x) \ No newline at end of file diff --git a/test.py b/test.py new file mode 100644 index 0000000..d7df59e --- /dev/null +++ b/test.py @@ -0,0 +1,159 @@ +import numpy as np +import nnfs +from nnfs.datasets import spiral_data +class neuron: + def __init__(self, weight: np.array, bias: float): + self.weight = weight + self.bias = bias + + def activate(self, input: np.array) -> int: + return np.dot(self.weight, input) + self.bias + + +class layer: + def __init__(self, weights: np.array, biases: np.array): + self.weights = np.array(weights) + self.biases = np.array(biases) + + def activate(self, inputs): + return np.dot(inputs, self.weights.T) + self.biases + +class layer_dense: + def __init__(self, n_inputs, n_neurons): + self.weights = 0.01 * np.random.randn(n_inputs, n_neurons) + self.biases = np.zeros((1, n_neurons)) + + def forward(self, inputs): + self.inputs = inputs + self.output = np.dot(inputs, self.weights) + self.biases + + def backward(self, dvalues): + self.dweights = np.dot(self.inputs.T, dvalues) + self.dbiases = np.sum(dvalues, axis=0, keepdims=True) + self.dinputs = np.dot(dvalues, self.weights.T) + +class activate_ReLU: + def forward(self, inputs): + self.output = np.maximum(0, inputs) + + def backward(self, dvalues): + self.dinputs = dvalues.copy() + self.dinputs[self.output <= 0] = 0 + +class activate_Softmax: + def forward(self, inputs): + exp_val = np.exp(inputs - np.max(inputs, axis=1, keepdims=True)) + base = np.sum(exp_val, axis=1, keepdims=True) + probs = exp_val / base + self.output = probs + + def backward(self, dvalues): + self.dinputs = np.empty_like(dvalues) + for index, (single_output, single_dvalues) in enumerate(zip(self.output, dvalues)): + single_output = single_output.reshape(-1, 1) + jacobian_matrix = np.diagflat(single_output) - np.dot(single_output, single_output.T) + self.dinputs[index] = np.dot(jacobian_matrix, single_dvalues) + +class Loss: + def calculate(self, output, y): + sample_losses = self.forward(output, y) + + data_loss = np.mean(sample_losses) + return data_loss + +class Categorical_CrossEntropy(Loss): + def forward(self, y_pred, y_true): + samples = len(y_pred) + y_pred_clipped = np.clip(y_pred, 1e-7, 1 - 1e-7) + + if len(y_true.shape) == 1: + correct_confidences = y_pred_clipped[ + range(samples), + y_true + ] + elif len(y_true.shape) == 2: + correct_confidences = np.sum( + y_pred_clipped * y_true, + axis = 1 + ) + negative_log_likelihoods = -np.log(correct_confidences) + return negative_log_likelihoods + + def backward(self, dvalues, y_true): + samples = len(dvalues) + labels = len(dvalues[0]) + + if len(y_true.shape) == 1: + y_true = np.eye(labels)[y_true] + + self.dinputs = -y_true / dvalues + self.dinputs = self.dinputs / samples + +class Softmax_Categorical_CrossEntropy: + def __init__(self): + self.activation = activate_Softmax() + self.loss = Categorical_CrossEntropy() + + def forward(self, inputs, y_true): + self.activation.forward(inputs) + self.output = self.activation.output + return self.loss.calculate(self.output, y_true) + + def backward(self, dvalues, y_true): + samples = len(dvalues) + + if len(y_true.shape) == 2: + y_true = np.argmax(y_true, axis=1) + + self.dinputs = dvalues.copy() + self.dinputs[range(samples), y_true] -= 1 + self.dinputs = self.dinputs / samples + +class Optimizer_SGD: + def __init__(self, learning_rate=0.5): + self.learning_rate = learning_rate + + def update_params(self, layer): + layer.weights += -self.learning_rate * layer.dweights + layer.biases += -self.learning_rate * layer.dbiases + +# Create input dataset +X, y = spiral_data(samples = 100, classes=3) + +# Define layers + functions +layer1 = layer_dense(2, 64) +activation1 = activate_ReLU() +layer2 = layer_dense(64, 3) +loss_activation = Softmax_Categorical_CrossEntropy() +optimizer = Optimizer_SGD() + +for epoch in range(10001): + # Forward pass + layer1.forward(X) + activation1.forward(layer1.output) + layer2.forward(activation1.output) + loss = loss_activation.forward(layer2.output, y) + + # # Print fp results + # print(loss_activation.output[:5]) + # print(f"loss: {loss}") + predictions = np.argmax(loss_activation.output, axis=1) + if len(y.shape) == 2: + y = np.argmax(y, axis=1) + accuracy = np.mean(predictions==y) + if not epoch % 100: + print(f"epoch: {epoch} acc: {accuracy} loss: {loss}") + # Backward pass + loss_backward = loss_activation.backward(loss_activation.output, y) + layer2.backward(loss_activation.dinputs) + activation1.backward(layer2.dinputs) + layer1.backward(activation1.dinputs) + + # print(layer1.dweights) + # print(layer1.dbiases) + # print(layer2.dweights) + # print(layer2.dbiases) + + # Update weights and biases + optimizer.update_params(layer1) + optimizer.update_params(layer2) \ No newline at end of file