runtime error

Exit code: 1. Reason: ained( File "/usr/local/lib/python3.10/site-packages/transformers/models/auto/auto_factory.py", line 597, in from_pretrained return model_class.from_pretrained( File "/usr/local/lib/python3.10/site-packages/transformers/modeling_utils.py", line 277, in _wrapper return func(*args, **kwargs) File "/usr/local/lib/python3.10/site-packages/transformers/modeling_utils.py", line 4971, in from_pretrained model = cls(config, *model_args, **model_kwargs) File "/home/user/.cache/huggingface/modules/transformers_modules/dots_hyphen_ocr_hyphen_local/modeling_dots_ocr.py", line 18, in __init__ super().__init__(config) File "/usr/local/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py", line 409, in __init__ super().__init__(config) File "/usr/local/lib/python3.10/site-packages/transformers/modeling_utils.py", line 2076, in __init__ self.config._attn_implementation_internal = self._check_and_adjust_attn_implementation( File "/usr/local/lib/python3.10/site-packages/transformers/modeling_utils.py", line 2691, in _check_and_adjust_attn_implementation lazy_import_flash_attention(applicable_attn_implementation, force_import=True) File "/usr/local/lib/python3.10/site-packages/transformers/modeling_flash_attention_utils.py", line 136, in lazy_import_flash_attention _flash_fn, _flash_varlen_fn, _pad_fn, _unpad_fn = _lazy_imports(implementation) File "/usr/local/lib/python3.10/site-packages/transformers/modeling_flash_attention_utils.py", line 83, in _lazy_imports from flash_attn import flash_attn_func, flash_attn_varlen_func File "/usr/local/lib/python3.10/site-packages/flash_attn/__init__.py", line 3, in <module> from flash_attn.flash_attn_interface import ( File "/usr/local/lib/python3.10/site-packages/flash_attn/flash_attn_interface.py", line 15, in <module> import flash_attn_2_cuda as flash_attn_gpu ImportError: /usr/local/lib/python3.10/site-packages/flash_attn_2_cuda.cpython-310-x86_64-linux-gnu.so: undefined symbol: _ZN3c104cuda9SetDeviceEa

Container logs: