diff --git a/torch_ac/algos/ppo.py b/torch_ac/algos/ppo.py
index b270d06..03425ab 100644
--- a/torch_ac/algos/ppo.py
+++ b/torch_ac/algos/ppo.py
@@ -103,8 +103,8 @@ def update_parameters(self, exps):
 
                 self.optimizer.zero_grad()
                 batch_loss.backward()
-                grad_norm = sum(p.grad.data.norm(2).item() ** 2 for p in self.acmodel.parameters()) ** 0.5
-                torch.nn.utils.clip_grad_norm_(self.acmodel.parameters(), self.max_grad_norm)
+                grad_norm = sum(p.grad.data.norm(2).item() ** 2 for p in self.acmodel.parameters() if p.requires_grad) ** 0.5
+                torch.nn.utils.clip_grad_norm_([p for p in self.acmodel.parameters() if p.requires_grad], self.max_grad_norm)
                 self.optimizer.step()
 
                 # Update log values