From 9d33700460ed7529dbeb89efa2846da2e490bec5 Mon Sep 17 00:00:00 2001 From: ConnorGoldsby Date: Sun, 5 Oct 2025 18:30:15 -0500 Subject: [PATCH 1/7] Add files via upload --- Neural-Networks-From-Scratch-Connor.zip | Bin 0 -> 980 bytes test.py | 1 + 2 files changed, 1 insertion(+) create mode 100644 Neural-Networks-From-Scratch-Connor.zip create mode 100644 test.py diff --git a/Neural-Networks-From-Scratch-Connor.zip b/Neural-Networks-From-Scratch-Connor.zip new file mode 100644 index 0000000000000000000000000000000000000000..6278333c6c66e3e4f65da381864887dd69fc3489 GIT binary patch literal 980 zcmWIWW@h1H00HM}*Jv;UN~kb!GWeyI7A5BB`lXhX=NDxc>$(-?=jsM07bTV?XXrZT z=jG)W>4%1}GBDPCc$5(UHHrgl)EmB;lHrUD3`?097<7p=D#+E*#n)9YHwA8HZP3QN zLk0qO|B9*`mShTTElFD2b!DN#)P>y4eZ>jgPv+bbSE#>Vtg%JmVA9RyY$Cn-%9ZO|?=wgDa7^6TdqB?Y#7Y*%XnEP+7o4XAU2jr< zQ7gXS;xVg9V)O4Mn0RjIgH1`B}T?*{&sC$AiA? zn)dk9EcRoTdnT{hzHZ?mv6W6neE)-YiQUmx;ugAH{+vn8-S9c{{LZ+d3TumLHF Date: Mon, 6 Oct 2025 00:38:50 +0000 Subject: [PATCH 2/7] test neuron --- test.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/test.py b/test.py index a3ea3c6..7191e73 100644 --- a/test.py +++ b/test.py @@ -1 +1,9 @@ -print("Hello") \ No newline at end of file +import numpy as np +class neuron: + def __init__(self, weight: np.array, bias: float): + self.weight = weight + self.bias = bias + + def activate(self, input: np.array) -> int: + return np.dot(self.weight, input) + self.bias + From affcd709b995faecbc17af59befd122001504d51 Mon Sep 17 00:00:00 2001 From: ConnorGoldsby Date: Mon, 13 Oct 2025 00:00:40 +0000 Subject: [PATCH 3/7] layers --- test.py | 30 +++++++++++++++++++++++++++++- 1 file changed, 29 insertions(+), 1 deletion(-) diff --git a/test.py b/test.py index 7191e73..04eefb5 100644 --- a/test.py +++ b/test.py @@ -3,7 +3,35 @@ class neuron: def __init__(self, weight: np.array, bias: float): self.weight = weight self.bias = bias - + def activate(self, input: np.array) -> int: return np.dot(self.weight, input) + self.bias + +class layer: + def __init__(self, weights, biases): + self.neurons = [] + for i in range(len(biases)): + self.neurons.append(neuron(weights[i], biases[i])) + + def activate(self, input): + output = [] + for i in self.neurons: + output.append(i.activate(input)) + return output + + + + + + + + +inputs = [1, 2, 3, 2.5] +weights = [[0.2, 0.8, -0.5, 1], [0.5, -0.91, 0.26, -0.5], [-0.26, -0.27, 0.17, 0.87]] +biases = [2, 3, 0.5] +neurons = layer(weights, biases) +outputs = neurons.activate(inputs) +print(outputs) + + From ae20d29fe41ee413c1126963ed3c2f4a9c2100d8 Mon Sep 17 00:00:00 2001 From: adurmusoglu Date: Sun, 12 Oct 2025 19:19:37 -0500 Subject: [PATCH 4/7] Add batch functionality --- test.py | 20 +++++++------------- 1 file changed, 7 insertions(+), 13 deletions(-) diff --git a/test.py b/test.py index 04eefb5..cf0aa6d 100644 --- a/test.py +++ b/test.py @@ -9,16 +9,12 @@ def activate(self, input: np.array) -> int: class layer: - def __init__(self, weights, biases): - self.neurons = [] - for i in range(len(biases)): - self.neurons.append(neuron(weights[i], biases[i])) + def __init__(self, weights: np.array, biases: np.array): + self.weights = np.array(weights) + self.biases = np.array(biases) - def activate(self, input): - output = [] - for i in self.neurons: - output.append(i.activate(input)) - return output + def activate(self, inputs): + return np.dot(inputs, self.weights.T) + self.biases @@ -27,11 +23,9 @@ def activate(self, input): -inputs = [1, 2, 3, 2.5] +inputs = [[1,2,3,4], [5,6,7,8], [9,10,11,12], [13,14,15,16]] weights = [[0.2, 0.8, -0.5, 1], [0.5, -0.91, 0.26, -0.5], [-0.26, -0.27, 0.17, 0.87]] biases = [2, 3, 0.5] neurons = layer(weights, biases) outputs = neurons.activate(inputs) -print(outputs) - - +print(outputs) \ No newline at end of file From 2e39d495f2388e01c86df40903122f54c0a04747 Mon Sep 17 00:00:00 2001 From: ConnorGoldsby Date: Mon, 13 Oct 2025 00:55:30 +0000 Subject: [PATCH 5/7] 2 layers --- test.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/test.py b/test.py index cf0aa6d..20ba8cd 100644 --- a/test.py +++ b/test.py @@ -28,4 +28,9 @@ def activate(self, inputs): biases = [2, 3, 0.5] neurons = layer(weights, biases) outputs = neurons.activate(inputs) -print(outputs) \ No newline at end of file +input = outputs +weights = [[0.2, 0.8, -0.5], [0.5, -0.91, 0.26], [-0.27, 0.17, 0.87]] +biases = [5, -0.8, 2] +hiddenLayer = layer(weights, biases) +ouputs = hiddenLayer.activate(input) +print(outputs) From 7671c6e74d4de66a8537f2f771e3bb9037b4b859 Mon Sep 17 00:00:00 2001 From: Kaushik Nagarajan Date: Sun, 16 Nov 2025 19:10:02 -0600 Subject: [PATCH 6/7] chapters 5-8 --- __pycache__/test.cpython-313.pyc | Bin 0 -> 6092 bytes ex.py | 60 ++++++++++++++++++++++++++ ex2.py | 53 +++++++++++++++++++++++ test.py | 71 +++++++++++++++++++++++++------ 4 files changed, 172 insertions(+), 12 deletions(-) create mode 100644 __pycache__/test.cpython-313.pyc create mode 100644 ex.py create mode 100644 ex2.py diff --git a/__pycache__/test.cpython-313.pyc b/__pycache__/test.cpython-313.pyc new file mode 100644 index 0000000000000000000000000000000000000000..87e02dfba5bb2c4540eefca8d63e6ff8ee38a244 GIT binary patch literal 6092 zcmbVQU2Gf25#A&3j-)95NTMj}Ck{m`l4)B?oJ2|N82*cECv~DWqLm6T5#~glC`Qzs zvPa668x#s!B&?g%vI9d@P*h4#!17DuJmsYid2U}s(1qlS3nXY>^wQdq13jiQd%V+G zrfVY$aBgpQc6N7WzS%i$_hxowvGx0;+*c) zrm|GZrADQ^l(uW-t1laGLFWodGH3uSIO`kPpluwT*^Rkxjo0`$xsZ>_U=2Iy> zV4S4SWMHDv>VeFXwQ$FW-nCHVBhLoB6J5wEQ-!=PQaN9sN*WJmHnamf=$p{FLe7|c zGH92oa${tO44NLZBncVf%z%b*(Kb-+_QJD^Q1oS-*B#1~?i^L~TQ&jn4TM*pSU!!)L#k2mydt`%C%R@f&LYD zKMjBoEP&o&D-OiHXhlVul);L1snn!8TF7C)K9zc>AmyqffmCWNOSOD1tH_F)O3^0J zpbHYE(u^*Q6VSt zzO&1Y7^1blEh+VIl^>>QF(43V zEv@tCE}vU+Ra$lx`48)Fg(LIhm&Z%fm2h{F|H#WcQyX26W{2TmEp*jFlnrkf*xQ;N zZ-QZm5Qs24=-g>$ifabubdlu-%?>*wn5%X$0oMo$OgmHoH@28t$So=hN+r;Ho&VDN zg|{4damD?j2|2VlTXP`pp)L3lLDGt(4GAL3@DN&_ElYP`?{Oqspuw7gB0({>2?I1e zOOC7VPu&Qbp6jtMcYU!7fz$H?0Oz1F9VeyXXEif*Z$yE$9VQU$JQiS7Nbpq52ys#O zvS>(+%8DkhBgpT$CJ==fc^e-xZ3*uG$Lt`=Z)?L=G1UvB_DHtrs^McO8yrK9XYClU zv}YjD2Qt47{q^Lx=YMy29nTyWjr=82Nf}iqbpfA>5fQ?9nW~x{Ai7(DYXVmpFUfFT1+;YqYoh5E*f9W@saNk4V zGY;AwemT{Kj}0mhxSGJ|JS@PPnLf6|hFM1!)j}SuLb24dBSv?Psr0NwNA)JVMXF4U zbqzxrQV67svCCtnH?Af>O;*BBUqAijnJ>(0B;BnOi4};8pl&S?8 zOaT-L#;k_2@00K0o_P|%ag;OAf#bE&fm^PP4k~2$xS;#YypS4{UmaS13^iGq8(z?$ z3>gQRCR;4qrCm>EZY zqM3u%NfPQ<9*(}wGMe3nSP3yp{~0OtHK_48_^5Iuoco=tSz2b6j$Y@Xo-Fcf!In$@qOcZj zuip3cC}$AE6XwvHnY3Hr?l zUw!Olo1=73TFRvhIV|w~c%(k-odLTG^uZ+(=~%2^s9$=c5_z)d`z{ooKY96N>9tB| zchR#Jh^)9HhCMOE4!}KEtI`;Tv6^}EbsRu~#6@QbG&StrkQf+uVdR^wAXMuv}J*^_W4g#8Xhuu0?B(zX^OzY?VphBz8evGmv@t^S^!a{p$HcnSZqX^6x+X_}{}U$*f~+LqRZ8V_ zK9yFLu`JYyX^8lUB4^+`a2g?yN#(K=axOcrs-qfGQ+ZS6tX;}&{hnCM(T3;yozw`Qw(-+_U{ks)!#}ZfZcCN*qx)p4m%Y5u# z3x;nyxrUZIPUlX4x&HCHZsKp43(t34>Ztg3EOl0VJ8#9hFbS>&ca(!YYb}vG0*J0K z(IcaIHtH8UVy859YIyoIiSu*;Z;BkQ9`F#6#m zJp&KjmCi|+H|U!33|?Vg452Lig`p}MeGA6v2olWB_}7MUfNi&i?mlSzA#sSl3vvij zZ3HNIkK?{3jb9V*zsM7t4v~BA4gs>+C~^m}cW^T(az{Dr9_2QJ4sP!q0%X%Ia2=xwGur`yM~b&F;JD3C+D;;wqlDS@EX7{SWG_ zca4Z2_}}vv-?>2=)`)kNG?huy+==bx#BLH{c4AeGl*LG?vDCRDcHAJ3-SqFM z_@g-06hE~Z>Mn=6Z;+mv6TWplUU7AJ1Y9#vi^mQveedjV56 Date: Sun, 23 Nov 2025 20:16:31 -0600 Subject: [PATCH 7/7] Banerjee, Devkumar - Add Backpropagataion, Rudimentary SGD Optimizer --- back.py | 51 ++++++++++++++++++++++++++ ex3.py | 4 +++ test.py | 110 +++++++++++++++++++++++++++++++++++++++++++++++--------- 3 files changed, 148 insertions(+), 17 deletions(-) create mode 100644 back.py create mode 100644 ex3.py diff --git a/back.py b/back.py new file mode 100644 index 0000000..6c4df3f --- /dev/null +++ b/back.py @@ -0,0 +1,51 @@ +x = [1.0, -2.0, 3.0] +w = [-3.0, -1.0, 2.0] +b = 1.0 + +xw0 = x[0] * w[0] +xw1 = x[1] * w[1] +xw2 = x[2] * w[2] + +z = xw0 + xw1 + xw2 + b +y = max(z, 0) + +dvalue = 1.0 +drelu_dz = dvalue * (1.0 if z > 0 else 0.0) + +dsum_dxw0 = 1 +drelu_dxw0 = drelu_dz * dsum_dxw0 + +dsum_dxw1 = 1 +drelu_dxw1 = drelu_dz * dsum_dxw1 + +dsum_dxw2 = 1 +drelu_dxw2 = drelu_dz * dsum_dxw2 + +dsum_db = 1 +drelu_db = drelu_dz * dsum_db + +# f = xy +# df/dx = y +# df/dy = x + +dmul_dx0 = w[0] +drelu_dx0 = drelu_dxw0 * dmul_dx0 + +dmul_dx1 = w[1] +drelu_dx1 = drelu_dxw1 * dmul_dx1 + +dmul_dx2 = w[2] +drelu_dx2 = drelu_dxw2 * dmul_dx2 + +dmul_dw0 = x[0] +drelu_dw0 = drelu_dxw0 * dmul_dw0 + +dmul_dw1 = x[1] +drelu_dw1 = drelu_dxw1 * dmul_dw1 + +dmul_dw2 = x[2] +drelu_dw2 = drelu_dxw2 * dmul_dw2 + +print(drelu_dx0, drelu_dx1, drelu_dx2) +print(drelu_dw0, drelu_dw1, drelu_dw2) +print(drelu_db) \ No newline at end of file diff --git a/ex3.py b/ex3.py new file mode 100644 index 0000000..9cc76a1 --- /dev/null +++ b/ex3.py @@ -0,0 +1,4 @@ +import numpy as np + +x = np.eye(5)[1] +print(x) \ No newline at end of file diff --git a/test.py b/test.py index b1f782c..d7df59e 100644 --- a/test.py +++ b/test.py @@ -24,11 +24,21 @@ def __init__(self, n_inputs, n_neurons): self.biases = np.zeros((1, n_neurons)) def forward(self, inputs): + self.inputs = inputs self.output = np.dot(inputs, self.weights) + self.biases + def backward(self, dvalues): + self.dweights = np.dot(self.inputs.T, dvalues) + self.dbiases = np.sum(dvalues, axis=0, keepdims=True) + self.dinputs = np.dot(dvalues, self.weights.T) + class activate_ReLU: def forward(self, inputs): self.output = np.maximum(0, inputs) + + def backward(self, dvalues): + self.dinputs = dvalues.copy() + self.dinputs[self.output <= 0] = 0 class activate_Softmax: def forward(self, inputs): @@ -36,6 +46,13 @@ def forward(self, inputs): base = np.sum(exp_val, axis=1, keepdims=True) probs = exp_val / base self.output = probs + + def backward(self, dvalues): + self.dinputs = np.empty_like(dvalues) + for index, (single_output, single_dvalues) in enumerate(zip(self.output, dvalues)): + single_output = single_output.reshape(-1, 1) + jacobian_matrix = np.diagflat(single_output) - np.dot(single_output, single_output.T) + self.dinputs[index] = np.dot(jacobian_matrix, single_dvalues) class Loss: def calculate(self, output, y): @@ -61,23 +78,82 @@ def forward(self, y_pred, y_true): ) negative_log_likelihoods = -np.log(correct_confidences) return negative_log_likelihoods + + def backward(self, dvalues, y_true): + samples = len(dvalues) + labels = len(dvalues[0]) + + if len(y_true.shape) == 1: + y_true = np.eye(labels)[y_true] + + self.dinputs = -y_true / dvalues + self.dinputs = self.dinputs / samples + +class Softmax_Categorical_CrossEntropy: + def __init__(self): + self.activation = activate_Softmax() + self.loss = Categorical_CrossEntropy() + + def forward(self, inputs, y_true): + self.activation.forward(inputs) + self.output = self.activation.output + return self.loss.calculate(self.output, y_true) + + def backward(self, dvalues, y_true): + samples = len(dvalues) + if len(y_true.shape) == 2: + y_true = np.argmax(y_true, axis=1) + + self.dinputs = dvalues.copy() + self.dinputs[range(samples), y_true] -= 1 + self.dinputs = self.dinputs / samples + +class Optimizer_SGD: + def __init__(self, learning_rate=0.5): + self.learning_rate = learning_rate + + def update_params(self, layer): + layer.weights += -self.learning_rate * layer.dweights + layer.biases += -self.learning_rate * layer.dbiases + +# Create input dataset X, y = spiral_data(samples = 100, classes=3) -layer1 = layer_dense(2, 3) + +# Define layers + functions +layer1 = layer_dense(2, 64) activation1 = activate_ReLU() -layer1.forward(X) -activation1.forward(layer1.output) -print(activation1.output[:5]) -activation2 = activate_Softmax() -activation2.forward(activation1.output) -print(activation2.output[:5]) - -loss = Categorical_CrossEntropy().calculate(activation2.output, y) -print(f"loss: {loss}") - -predictions = np.argmax(activation2.output, axis=1) -if len(y.shape) == 2: - y = np.argmax(y, axis=1) - -accuracy = np.mean(predictions == y) -print(f"accuracy: {accuracy}") +layer2 = layer_dense(64, 3) +loss_activation = Softmax_Categorical_CrossEntropy() +optimizer = Optimizer_SGD() + +for epoch in range(10001): + # Forward pass + layer1.forward(X) + activation1.forward(layer1.output) + layer2.forward(activation1.output) + loss = loss_activation.forward(layer2.output, y) + + # # Print fp results + # print(loss_activation.output[:5]) + # print(f"loss: {loss}") + predictions = np.argmax(loss_activation.output, axis=1) + if len(y.shape) == 2: + y = np.argmax(y, axis=1) + accuracy = np.mean(predictions==y) + if not epoch % 100: + print(f"epoch: {epoch} acc: {accuracy} loss: {loss}") + # Backward pass + loss_backward = loss_activation.backward(loss_activation.output, y) + layer2.backward(loss_activation.dinputs) + activation1.backward(layer2.dinputs) + layer1.backward(activation1.dinputs) + + # print(layer1.dweights) + # print(layer1.dbiases) + # print(layer2.dweights) + # print(layer2.dbiases) + + # Update weights and biases + optimizer.update_params(layer1) + optimizer.update_params(layer2) \ No newline at end of file