When I using san model to train my dataset, ValueError: matrix contains invalid numeric entries

[fade54188]

Traceback (most recent call last):
File "tools/train.py", line 104, in
main()
File "tools/train.py", line 100, in main
runner.train()
File "/root/miniconda3/lib/python3.8/site-packages/mmengine/runner/runner.py", line 1706, in train
model = self.train_loop.run() # type: ignore
File "/root/miniconda3/lib/python3.8/site-packages/mmengine/runner/loops.py", line 278, in run
self.run_iter(data_batch)
File "/root/miniconda3/lib/python3.8/site-packages/mmengine/runner/loops.py", line 301, in run_iter
outputs = self.runner.model.train_step(
File "/root/miniconda3/lib/python3.8/site-packages/mmengine/model/base_model/base_model.py", line 114, in train_step
losses = self._run_forward(data, mode='loss') # type: ignore
File "/root/miniconda3/lib/python3.8/site-packages/mmengine/model/base_model/base_model.py", line 326, in _run_forward
results = self(**data, mode=mode)
File "/root/miniconda3/lib/python3.8/site-packages/torch/nn/modules/module.py", line 1102, in _call_impl
return forward_call(*input, **kwargs)
File "/root/autodl-tmp/mmsegmentation/mmseg/models/segmentors/base.py", line 94, in forward
return self.loss(inputs, data_samples)
File "/root/autodl-tmp/mmsegmentation/mmseg/models/segmentors/multimodal_encoder_decoder.py", line 172, in loss
loss_decode = self._decode_head_forward_train(
File "/root/autodl-tmp/mmsegmentation/mmseg/models/segmentors/multimodal_encoder_decoder.py", line 145, in _decode_head_forward_train
loss_decode = self.decode_head.loss(inputs, data_samples,
File "/root/autodl-tmp/mmsegmentation/mmseg/models/decode_heads/san_head.py", line 630, in loss
losses = self.loss_by_feat(all_mask_logits, all_mask_props,
File "/root/autodl-tmp/mmsegmentation/mmseg/models/decode_heads/san_head.py", line 664, in loss_by_feat
avg_factor) = self.match_masks.get_targets(
File "/root/autodl-tmp/mmsegmentation/mmseg/utils/mask_classification.py", line 119, in get_targets
= self._get_targets_single(cls_scores[i],
File "/root/autodl-tmp/mmsegmentation/mmseg/utils/mask_classification.py", line 193, in _get_targets_single
matched_quiery_inds, matched_label_inds = self.assigner.assign(
File "/root/autodl-tmp/mmsegmentation/mmseg/models/assigners/hungarian_assigner.py", line 82, in assign
matched_quiery_inds, matched_label_inds = linear_sum_assignment(cost)
ValueError: matrix contains invalid numeric entries