Training code/compatibility (Please check the below code and tell me why i am getting the given issue)

[Safiislamian]

unpacking data ....
data length is 250
augmentation: True
check model parameter: 2.398951530456543, keep model parameters on different processes consistent
model parameters is 139.214584M
step number is 250000
initial lr is 0.01
/root/miniconda3/envs/mri/lib/python3.10/site-packages/torch/optim/lr_scheduler.py:62: UserWarning: The verbose parameter is deprecated. Please use get_last_lr() to access the learning rate.
warnings.warn(
scheduler_type is poly, warmup steps is 0
Epoch 0: 0%| | 0/250 [00:00<?, ?it/s]using pin_memory on device 0
Image shape after padding: torch.Size([2, 4, 128, 128, 128])
Epoch 0: 0%| | 0/250 [00:18<?, ?it/s]
Traceback (most recent call last):
File "/root/SegMamba-V2/brats23/3_train.py", line 217, in
trainer.train(train_dataset=train_ds, val_dataset=test_ds)
File "/root/SegMamba-V2/brats23/light_training/trainer.py", line 407, in train
self.train_epoch(
File "/root/SegMamba-V2/brats23/light_training/trainer.py", line 450, in train_epoch
loss = self.training_step(batch)
File "/root/SegMamba-V2/brats23/3_train.py", line 111, in training_step
pred = self.model(image)
File "/root/miniconda3/envs/mri/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1736, in _wrapped_call_impl
return self._call_impl(*args, **kwargs)
File "/root/miniconda3/envs/mri/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1747, in _call_impl
return forward_call(*args, **kwargs)
File "/root/SegMamba-V2/brats23/models_segmamba/segmambav2.py", line 406, in forward
outs = self.vit(enc1)
File "/root/miniconda3/envs/mri/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1736, in _wrapped_call_impl
return self._call_impl(*args, **kwargs)
File "/root/miniconda3/envs/mri/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1747, in _call_impl
return forward_call(*args, **kwargs)
File "/root/SegMamba-V2/brats23/models_segmamba/segmambav2.py", line 265, in forward
x = self.forward_features(x)
File "/root/SegMamba-V2/brats23/models_segmamba/segmambav2.py", line 254, in forward_features
x = self.stagesi
File "/root/miniconda3/envs/mri/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1736, in _wrapped_call_impl
return self._call_impl(*args, **kwargs)
File "/root/miniconda3/envs/mri/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1747, in _call_impl
return forward_call(*args, **kwargs)
File "/root/miniconda3/envs/mri/lib/python3.10/site-packages/torch/nn/modules/container.py", line 250, in forward
input = module(input)
File "/root/miniconda3/envs/mri/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1736, in _wrapped_call_impl
return self._call_impl(*args, **kwargs)
File "/root/miniconda3/envs/mri/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1747, in _call_impl
return forward_call(*args, **kwargs)
File "/root/SegMamba-V2/brats23/models_segmamba/segmambav2.py", line 49, in forward
out_x_1 = self.mamba_forward(x)
File "/root/SegMamba-V2/brats23/models_segmamba/segmambav2.py", line 40, in mamba_forward
x = self.mamba(x)
File "/root/miniconda3/envs/mri/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1736, in _wrapped_call_impl
return self._call_impl(*args, **kwargs)
File "/root/miniconda3/envs/mri/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1747, in _call_impl
return forward_call(*args, **kwargs)
File "/root/miniconda3/envs/mri/lib/python3.10/site-packages/mamba_ssm-1.0.1-py3.10-linux-x86_64.egg/mamba_ssm/modules/mamba_simple.py", line 246, in forward
xz_s = torch.stack(xz_s,dim=-1)
RuntimeError: stack expects each tensor to be equal size, but got [2, 768, 820] at entry 0 and [2, 768, 816] at entry 4
Exception in thread Thread-9 (results_loop):
Traceback (most recent call last):
File "/root/miniconda3/envs/mri/lib/python3.10/threading.py", line 1016, in _bootstrap_inner
self.run()
File "/root/miniconda3/envs/mri/lib/python3.10/threading.py", line 953, in run
self._target(*self._args, **self._kwargs)
File "/root/miniconda3/envs/mri/lib/python3.10/site-packages/batchgenerators/dataloading/nondet_multi_threaded_augmenter.py", line 125, in results_loop
raise e
File "/root/miniconda3/envs/mri/lib/python3.10/site-packages/batchgenerators/dataloading/nondet_multi_threaded_augmenter.py", line 103, in results_loop
raise RuntimeError("One or more background workers are no longer alive. Exiting. Please check the "
RuntimeError: One or more background workers are no longer alive. Exiting. Please check the print statements above for the actual error message
Exception ignored in: <function NonDetMultiThreadedAugmenter.del at 0x7cb5943fa680>
Traceback (most recent call last):
File "/root/miniconda3/envs/mri/lib/python3.10/site-packages/batchgenerators/dataloading/nondet_multi_threaded_augmenter.py", line 259, in del
File "/root/miniconda3/envs/mri/lib/python3.10/site-packages/batchgenerators/dataloading/nondet_multi_threaded_augmenter.py", line 244, in _finish
File "/root/miniconda3/envs/mri/lib/python3.10/multiprocessing/synchronize.py", line 338, in set
File "/root/miniconda3/envs/mri/lib/python3.10/multiprocessing/synchronize.py", line 297, in notify_all
AttributeError: 'NoneType' object has no attribute 'maxsize'