Thanks for the excellent work. May I ask how the following issue can be resolved?

[yqytomorrow]

Traceback (most recent call last):
File "/mnt/data3/yqy/projects/Human3R/demo.py", line 692, in
main()
File "/mnt/data3/yqy/projects/Human3R/demo.py", line 688, in main
run_inference(args)
File "/mnt/data3/yqy/projects/Human3R/demo.py", line 623, in run_inference
outputs, _ = inference_recurrent_lighter(
^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/mnt/data3/yqy/tools/miniconda3/envs/human3r/lib/python3.11/site-packages/torch/utils/_contextlib.py", line 116, in decorate_context
return func(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^
File "/mnt/data3/yqy/projects/Human3R/src/dust3r/inference.py", line 174, in inference_recurrent_lighter
preds, batch, state_args = model.forward_recurrent_lighter(
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/mnt/data3/yqy/projects/Human3R/src/dust3r/model.py", line 1559, in forward_recurrent_lighter
new_state_feat, dec, cross_attn_states = self._recurrent_rollout(
^^^^^^^^^^^^^^^^^^^^^^^^
File "/mnt/data3/yqy/projects/Human3R/src/dust3r/model.py", line 933, in _recurrent_rollout
(new_state_feat, dec), cross_attn_states = self._decoder(
^^^^^^^^^^^^^^
File "/mnt/data3/yqy/projects/Human3R/src/dust3r/model.py", line 890, in _decoder
f_state, _, cross_attn_state = blk_state(*final_output[-1][::+1], pos_state, pos_img, use_ttt3r=use_ttt3r)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/mnt/data3/yqy/tools/miniconda3/envs/human3r/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1736, in _wrapped_call_impl
return self.call_impl(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/mnt/data3/yqy/tools/miniconda3/envs/human3r/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1747, in call_impl
return forward_call(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/mnt/data3/yqy/projects/Human3R/src/dust3r/blocks.py", line 309, in forward
cross_attn_output, cross_attn = self.cross_attn(self.norm2(x), y, y, xpos, ypos, use_ttt3r=use_ttt3r)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/mnt/data3/yqy/tools/miniconda3/envs/human3r/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1736, in _wrapped_call_impl
return self._call_impl(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/mnt/data3/yqy/tools/miniconda3/envs/human3r/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1747, in _call_impl
return forward_call(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/mnt/data3/yqy/projects/Human3R/src/dust3r/blocks.py", line 235, in forward
attn = (q @ k.transpose(-2, -1)) * self.scale # [B, num_heads, Nq, Nk] [1, 16, 768, 1 + 576]
~~^~~~~~~~~~~~~~~~~~~~~
RuntimeError: CUDA error: CUBLAS_STATUS_INTERNAL_ERROR when calling cublasSgemmStridedBatched( handle, opa, opb, m, n, k, &alpha, a, lda, stridea, b, ldb, strideb, &beta, c, ldc, stridec, num_batches)

[fanegg]

Hi @yqytomorrow, this may be related to the similar issue in Inception3D/TTT3R#14