From 2ac8cc3f9f646af047f5f5f8e7a18f0cb56050ba Mon Sep 17 00:00:00 2001 From: Jun Kim Date: Mon, 28 Oct 2024 22:26:58 +0900 Subject: [PATCH 1/2] Fix spelling error --- makeMoE_from_Scratch.ipynb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/makeMoE_from_Scratch.ipynb b/makeMoE_from_Scratch.ipynb index 428d6b7..6c21faa 100644 --- a/makeMoE_from_Scratch.ipynb +++ b/makeMoE_from_Scratch.ipynb @@ -1393,8 +1393,8 @@ " self.top_k = top_k\n", " self.linear =nn.Linear(n_embed, num_experts)\n", "\n", - " def forward(self, mh_ouput):\n", - " # mh_ouput is the output tensor from multihead self attention block\n", + " def forward(self, mh_output):\n", + " # mh_output is the output tensor from multihead self attention block\n", " logits = self.linear(mh_output)\n", " top_k_logits, indices = logits.topk(self.top_k, dim=-1)\n", " zeros = torch.full_like(logits, float('-inf'))\n", From 602863b9e7e7d29fa3ab73bb28f0b30aefc8368c Mon Sep 17 00:00:00 2001 From: Jun Kim Date: Mon, 28 Oct 2024 22:29:06 +0900 Subject: [PATCH 2/2] Fix spelling error --- makeMoE_Concise.ipynb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/makeMoE_Concise.ipynb b/makeMoE_Concise.ipynb index 6411f8a..1863781 100644 --- a/makeMoE_Concise.ipynb +++ b/makeMoE_Concise.ipynb @@ -441,7 +441,7 @@ "\n", " \n", " def forward(self, mh_output):\n", - " # mh_ouput is the output tensor from multihead self attention block\n", + " # mh_output is the output tensor from multihead self attention block\n", " logits = self.topkroute_linear(mh_output)\n", "\n", " #Noise logits\n",