Hi,
I’m attempting to run on a workstation. this job fails to connect with the following error:
Please post the text of the Traceback (inside the body of your post) so that question can be found more easily by interested future visitors to the forum.
Please also post the output of these commands.
Traceback (most recent call last):? File
“cryosparc_master/cryosparc_compute/run.py”, line 83, in
cryosparc_compute.run.main? File
“/home/amax/cryosparc/cryosparc_worker/cryosparc_compute/jobs/jobreg
ister.py”, line 442, in get_run_function?
runmod = importlib.impo
rt_module(“…”+modname, name)? File “/home/amax/cryosparc/cryos
parc_worker/deps/anaconda/envs/cryosparc_worker_env/lib/python3.8/im
portlib/init.py”, line 127, in import_module?
return _bootstr
ap._gcd_import(name[level:], package, level)? File “”, line 1014, in _gcd_import? File “<frozen importlib
._bootstrap>”, line 991, in _find_and_load? File “<frozen importlib
._bootstrap>”, line 975, in _find_and_load_unlocked? File “”, line 671, in _load_unlocked? File “”, line 1174, in exec_module? File “<
frozen importlib._bootstrap>”, line 219, in call_with_frames_remove
d? File “cryosparc_master/cryosparc_compute/jobs/var3D/run.py”, lin
e 18, in init cryosparc_compute.jobs.var3D.run? File “/home/amax/cr
yosparc/cryosparc_worker/cryosparc_compute/engine/init.py”, line
8, in ?
from .engine import * # noqa? File "cryosparc
master/cryosparc_compute/engine/engine.py", line 9, in init cryospar
c_compute.engine.engine? File “cryosparc_master/cryosparc_compute/e
ngine/cuda_core.py”, line 4, in init cryosparc_compute.engine.cuda_c
ore? File “/home/amax/cryosparc/cryosparc_worker/deps/anaconda/envs
/cryosparc_worker_env/lib/python3.8/site-packages/pycuda/driver.py”,
line 62, in ?
from pycuda._driver import * # noqa?Impor
tError: /home/amax/cryosparc/cryosparc_worker/deps/anaconda/envs/cry
osparc_worker_env/lib/python3.8/site-packages/pycuda/_driver.cpython
-38-x86_64-linux-gnu.so: undefined symbol: cuIpcOpenMemHandle_v2
================= CRYOSPARCW ======= 2023-04-12 09:08:22.576725 =========
Project P32 Job J212
Master localhost Port 60122
===========================================================================
========= monitor process now starting main process at 2023-04-12 09:08:22.576785
MAINPROCESS PID 13098
/home/amax/cryosparc/cryosparc_worker/deps/anaconda/envs/cryosparc_worker_env/lib/python3.8/site-packages/pycuda/driver.py:66: UserWarning: Failed to import the CUDA driver interface, with an error message indicating that the version of your CUDA header does not match the version of your CUDA driver.
warn("Failed to import the CUDA driver interface, with an error "
Traceback (most recent call last):
File "<string>", line 1, in <module>
File "cryosparc_master/cryosparc_compute/run.py", line 184, in cryosparc_compute.run.run
File "/home/amax/cryosparc/cryosparc_worker/cryosparc_compute/jobs/runcommon.py", line 2178, in get_gpu_info
import pycuda.driver as cudrv
File "/home/amax/cryosparc/cryosparc_worker/deps/anaconda/envs/cryosparc_worker_env/lib/python3.8/site-packages/pycuda/driver.py", line 62, in <module>
from pycuda._driver import * # noqa
ImportError: /home/amax/cryosparc/cryosparc_worker/deps/anaconda/envs/cryosparc_worker_env/lib/python3.8/site-packages/pycuda/_driver.cpython-38-x86_64-linux-gnu.so: undefined symbol: cuIpcOpenMemHandle_v2
libgcc_s.so.1 must be installed for pthread_cancel to work
MAIN PID 13098
var3D.run cryosparc_compute.jobs.jobregister
/home/amax/cryosparc/cryosparc_worker/bin/cryosparcw: line 165: 13097 Aborted (core dumped) python -c "import cryosparc_compute.run as run; run.run()" "$@"
**** handle exception rc
/home/amax/cryosparc/cryosparc_worker/deps/anaconda/envs/cryosparc_worker_env/lib/python3.8/site-packages/pycuda/driver.py:66: UserWarning: Failed to import the CUDA driver interface, with an error message indicating that the version of your CUDA header does not match the version of your CUDA driver.
warn("Failed to import the CUDA driver interface, with an error "
Traceback (most recent call last):
File "cryosparc_master/cryosparc_compute/run.py", line 83, in cryosparc_compute.run.main
File "/home/amax/cryosparc/cryosparc_worker/cryosparc_compute/jobs/jobregister.py", line 442, in get_run_function
runmod = importlib.import_module(".."+modname, __name__)
File "/home/amax/cryosparc/cryosparc_worker/deps/anaconda/envs/cryosparc_worker_env/lib/python3.8/importlib/__init__.py", line 127, in import_module
return _bootstrap._gcd_import(name[level:], package, level)
File "<frozen importlib._bootstrap>", line 1014, in _gcd_import
File "<frozen importlib._bootstrap>", line 991, in _find_and_load
File "<frozen importlib._bootstrap>", line 975, in _find_and_load_unlocked
File "<frozen importlib._bootstrap>", line 671, in _load_unlocked
File "<frozen importlib._bootstrap_external>", line 1174, in exec_module
File "<frozen importlib._bootstrap>", line 219, in _call_with_frames_removed
File "cryosparc_master/cryosparc_compute/jobs/var3D/run.py", line 18, in init cryosparc_compute.jobs.var3D.run
File "/home/amax/cryosparc/cryosparc_worker/cryosparc_compute/engine/__init__.py", line 8, in <module>
from .engine import * # noqa
File "cryosparc_master/cryosparc_compute/engine/engine.py", line 9, in init cryosparc_compute.engine.engine
File "cryosparc_master/cryosparc_compute/engine/cuda_core.py", line 4, in init cryosparc_compute.engine.cuda_core
File "/home/amax/cryosparc/cryosparc_worker/deps/anaconda/envs/cryosparc_worker_env/lib/python3.8/site-packages/pycuda/driver.py", line 62, in <module>
from pycuda._driver import * # noqa
ImportError: /home/amax/cryosparc/cryosparc_worker/deps/anaconda/envs/cryosparc_worker_env/lib/python3.8/site-packages/pycuda/_driver.cpython-38-x86_64-linux-gnu.so: undefined symbol: cuIpcOpenMemHandle_v2
set status to failed
Thanks for posting the text of the error messages.
Please also post outputs according to these instructions: