Traceback (most recent call last): File "/opt/chainguard/pytorch/lib/python3.11/site-packages/torch/__init__.py", line 433, in _load_global_deps ctypes.CDLL(global_deps_lib_path, mode=ctypes.RTLD_GLOBAL) File "/usr/lib/python3.11/ctypes/__init__.py", line 376, in __init__ self._handle = _dlopen(self._name, mode) ^^^^^^^^^^^^^^^^^^^^^^^^^ OSError: libcudart.so.12: cannot open shared object file: No such file or directory During handling of the above exception, another exception occurred: Traceback (most recent call last): File "/opt/chainguard/pytorch/bin/torchrun", line 4, in from torch.distributed.run import main File "/opt/chainguard/pytorch/lib/python3.11/site-packages/torch/__init__.py", line 504, in _load_global_deps() File "/opt/chainguard/pytorch/lib/python3.11/site-packages/torch/__init__.py", line 462, in _load_global_deps _preload_cuda_deps(err) File "/opt/chainguard/pytorch/lib/python3.11/site-packages/torch/__init__.py", line 395, in _preload_cuda_deps _preload_cuda_lib(lib_folder, lib_name, required=required) File "/opt/chainguard/pytorch/lib/python3.11/site-packages/torch/__init__.py", line 356, in _preload_cuda_lib raise ValueError(f"{lib_name} not found in the system path {sys.path}") ValueError: libcublasLt.so.*[0-9] not found in the system path ['/opt/chainguard/pytorch/bin', '/usr/lib/python311.zip', '/usr/lib/python3.11', '/usr/lib/python3.11/lib-dynload', '/opt/chainguard/pytorch/lib/python3.11/site-packages']