Fix 7 bugs found in deep code review: models, env wrappers, train loop

scripts/env_utils.py

@@ -11,10 +11,20 @@
 from utils import resolve_clip_reward, resolve_sticky_action_settings
 
 
+_MLP_POLICIES = frozenset({
+    'MlpPolicy',
+    'MlpDropoutPolicy',
+    'CombinedPolicy',
+    'AttentionMLPPolicy',
+    'EntityAttentionPolicy',
+    'HockeyMultiHeadPolicy',
+    'HybridMambaPolicy',
+    'GRUMlpPolicy',
+})
+
+
 def isMLP(name):
-    return name == 'MlpPolicy' or name == 'MlpDropoutPolicy' or name == 'CombinedPolicy' \
-          or name == 'AttentionMLPPolicy' or name == 'EntityAttentionPolicy' or name == 'HockeyMultiHeadPolicy' \
-          or name == 'HybridMambaPolicy' or name == 'GRUMlpPolicy'
+    return name in _MLP_POLICIES
 
 
 def make_retro(

scripts/env_wrappers.py

@@ -69,7 +69,7 @@ def observation(self, obs):
         else:
             img = cv2.resize(img, (self.width, self.height), interpolation=cv2.INTER_AREA)
             # Convert back to CHW
-            #img = np.transpose(img, (2, 0, 1))
+            img = np.transpose(img, (2, 0, 1))
 
         img = img.astype(np.uint8)
 

scripts/models.py

@@ -278,21 +278,18 @@ def __init__(self, observation_space, features_dim=64, hidden_dim=128):
     def forward(self, x):
         if x.dim() == 1:
             x = x.unsqueeze(0)
-        # Self-attention
+        # Self-attention: compute element-wise attention weights across all features
         q = self.query(x)  # [batch_size, num_features]
         k = self.key(x)    # [batch_size, num_features]
 
-        # Compute attention scores
-        # q.unsqueeze(1): [batch_size, 1, num_features]
-        # k.unsqueeze(2): [batch_size, num_features, 1]
-        # bmm result: [batch_size, 1, 1]
-        attention_scores = torch.bmm(q.unsqueeze(1), k.unsqueeze(2)) / (self.num_features ** 0.5)
+        # Element-wise product gives a per-feature attention score, then
+        # softmax normalises across the feature dimension so that the weights
+        # sum to 1 and every feature dimension gets its own importance score.
+        attention_scores = (q * k) / (self.num_features ** 0.5)  # [batch_size, num_features]
+        weights = torch.softmax(attention_scores, dim=-1)         # [batch_size, num_features]
 
-        # Apply softmax - need to specify dimension
-        weights = torch.softmax(attention_scores, dim=-1)  # Apply along last dimension
-
-        # Apply attention weights
-        attended_x = weights.squeeze(-1) * x  # [batch_size, num_features]
+        # Scale input features by their attention weights
+        attended_x = weights * x  # [batch_size, num_features]
 
         # MLP
         return self.mlp(attended_x)
@@ -392,7 +389,6 @@ def __init__(
         d_state=64,
         d_conv=4,
         expand=2,
-        gru_layers=2,
     ):
         self.d_state = d_state
         self.d_conv = d_conv
@@ -575,12 +571,12 @@ def __init__(self, observation_space, action_space, lr_schedule, net_arch=None,
             nn.ReLU()
         )
 
-        # Value head should output a single value
+        # Value head should output a single value (no activation after final linear
+        # — value estimates can be negative)
         self.value_net = nn.Sequential(
             nn.Linear(latent_dim_vf, latent_dim_vf),
             nn.ReLU(),
-            nn.Linear(latent_dim_vf, 1),  # Output single value
-            nn.ReLU()
+            nn.Linear(latent_dim_vf, 1),
         )
 
     def forward(self, obs, deterministic=False):
@@ -824,8 +820,9 @@ def __init__(self, feature_dim, net_arch, dropout_prob=0.5):
         )
 
     def forward(self, features):
-        # Return policy and value features for the base policy's use
-        return self.forward_actor(features), self.forward_critic(features)
+        # Run shared layers once and reuse for both heads
+        shared = self.shared_net(features)
+        return self.policy_net(shared), self.value_net(shared)
 
     def forward_actor(self, features):
         shared = self.shared_net(features)

scripts/train.py

@@ -102,7 +102,7 @@ def play(self, args, continuous=True):
 
         state = self.env.reset()
         while True:
-            self.env.render(mode='human')
+            self.env.render()
 
             p1_actions = self.p1_model.predict(state, deterministic=args.deterministic)