Skip to content

Commit ae77a48

Browse files
committed
Fix TransformerPath positional encoding and aux_weights docs
- Add learnable positional embeddings to TransformerPath for proper spatial reasoning - Implement dynamic positional embedding interpolation for varying input sizes - Add positional dropout for regularization - Update aux_weights docstring to clarify it's for external use only Addresses CodeRabbit review comments on PR Project-MONAI#8717 Signed-off-by: Sefa Aras <sefa666@hotmail.com>
1 parent 7330785 commit ae77a48

1 file changed

Lines changed: 45 additions & 2 deletions

File tree

monai/networks/nets/magnus.py

Lines changed: 45 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -115,7 +115,8 @@ class TransformerPath(nn.Module):
115115
Vision Transformer path for global context modeling.
116116
117117
Applies patch embedding followed by transformer encoder layers
118-
to capture long-range dependencies.
118+
to capture long-range dependencies. Includes learnable positional
119+
embeddings that are interpolated to match varying input sizes.
119120
120121
Args:
121122
spatial_dims: number of spatial dimensions (2 or 3).
@@ -150,6 +151,14 @@ def __init__(
150151
in_channels, hidden_dim, kernel_size=patch_size, stride=patch_size
151152
)
152153

154+
# Learnable positional embedding (will be interpolated for different input sizes)
155+
# Initialize with a reasonable default size, will adapt dynamically
156+
self.pos_embed = nn.Parameter(torch.zeros(1, 256, hidden_dim))
157+
nn.init.trunc_normal_(self.pos_embed, std=0.02)
158+
159+
# Dropout for positional embedding
160+
self.pos_drop = nn.Dropout(p=dropout)
161+
153162
# Transformer encoder
154163
encoder_layer = nn.TransformerEncoderLayer(
155164
d_model=hidden_dim,
@@ -165,6 +174,31 @@ def __init__(
165174
# Layer normalization
166175
self.norm = nn.LayerNorm(hidden_dim)
167176

177+
def _interpolate_pos_encoding(self, x: torch.Tensor, num_patches: int) -> torch.Tensor:
178+
"""
179+
Interpolate positional embeddings to match the number of patches.
180+
181+
Args:
182+
x: input tensor for device reference.
183+
num_patches: target number of patches.
184+
185+
Returns:
186+
Interpolated positional embeddings of shape (1, num_patches, hidden_dim).
187+
"""
188+
if num_patches == self.pos_embed.shape[1]:
189+
return self.pos_embed
190+
191+
# Interpolate positional embeddings
192+
pos_embed = self.pos_embed.transpose(1, 2) # (1, hidden_dim, N)
193+
pos_embed = F.interpolate(
194+
pos_embed,
195+
size=num_patches,
196+
mode="linear",
197+
align_corners=False,
198+
)
199+
pos_embed = pos_embed.transpose(1, 2) # (1, num_patches, hidden_dim)
200+
return pos_embed
201+
168202
def forward(self, x: torch.Tensor) -> torch.Tensor:
169203
"""
170204
Forward pass through transformer path.
@@ -182,6 +216,12 @@ def forward(self, x: torch.Tensor) -> torch.Tensor:
182216

183217
# Flatten spatial dims: (B, hidden_dim, *spatial) -> (B, N, hidden_dim)
184218
x_flat = x_embedded.flatten(2).transpose(1, 2)
219+
num_patches = x_flat.shape[1]
220+
221+
# Add positional encoding
222+
pos_embed = self._interpolate_pos_encoding(x_flat, num_patches)
223+
x_flat = x_flat + pos_embed
224+
x_flat = self.pos_drop(x_flat)
185225

186226
# Apply transformer
187227
x_transformed = self.transformer(x_flat)
@@ -512,7 +552,10 @@ class MAGNUS(nn.Module):
512552
dropout: dropout ratio. Default: 0.0.
513553
vit_dropout: dropout ratio for transformer. Default: 0.1.
514554
deep_supervision: whether to return auxiliary outputs. Default: False.
515-
aux_weights: weights for auxiliary losses. Default: (0.4, 0.3, 0.3).
555+
aux_weights: suggested weights for auxiliary losses when using deep supervision.
556+
These weights are stored as an attribute for user convenience but are NOT
557+
applied internally. Users should apply them externally when computing the
558+
total loss. Default: (0.4, 0.3, 0.3).
516559
517560
Example:
518561
>>> import torch

0 commit comments

Comments
 (0)