diff --git a/makeMoE_Concise.ipynb b/makeMoE_Concise.ipynb index 6411f8a..1863781 100644 --- a/makeMoE_Concise.ipynb +++ b/makeMoE_Concise.ipynb @@ -441,7 +441,7 @@ "\n", " \n", " def forward(self, mh_output):\n", - " # mh_ouput is the output tensor from multihead self attention block\n", + " # mh_output is the output tensor from multihead self attention block\n", " logits = self.topkroute_linear(mh_output)\n", "\n", " #Noise logits\n", diff --git a/makeMoE_from_Scratch.ipynb b/makeMoE_from_Scratch.ipynb index 428d6b7..6c21faa 100644 --- a/makeMoE_from_Scratch.ipynb +++ b/makeMoE_from_Scratch.ipynb @@ -1393,8 +1393,8 @@ " self.top_k = top_k\n", " self.linear =nn.Linear(n_embed, num_experts)\n", "\n", - " def forward(self, mh_ouput):\n", - " # mh_ouput is the output tensor from multihead self attention block\n", + " def forward(self, mh_output):\n", + " # mh_output is the output tensor from multihead self attention block\n", " logits = self.linear(mh_output)\n", " top_k_logits, indices = logits.topk(self.top_k, dim=-1)\n", " zeros = torch.full_like(logits, float('-inf'))\n",