(base) [cryosparc_user@localhost ~]$ csprojectid=P17
(base) [cryosparc_user@localhost ~]$ csjobid=J200
(base) [cryosparc_user@localhost ~]$ cryosparcm cli "get_job('$csprojectid', '$csjobid', 'job_type', 'version', 'instance_information', 'status', 'params_spec', 'errors_run')"
{'_id': 'xxx', 'errors_run': [{'message': '', 'warning': False}], 'instance_information': {'CUDA_version': '11.8', 'available_memory': '116.50GB', 'cpu_model': 'Intel(R) Xeon(R) CPU E5-2640 v4 @ 2.40GHz', 'driver_version': '12.4', 'gpu_info': [{'id': 0, 'mem': 11707547648, 'name': 'NVIDIA GeForce GTX 1080 Ti', 'pcie': '0000:02:00'}, {'id': 1, 'mem': 11707547648, 'name': 'NVIDIA GeForce GTX 1080 Ti', 'pcie': '0000:03:00'}, {'id': 2, 'mem': 11707547648, 'name': 'NVIDIA GeForce GTX 1080 Ti', 'pcie': '0000:82:00'}, {'id': 3, 'mem': 11707547648, 'name': 'NVIDIA GeForce GTX 1080 Ti', 'pcie': '0000:83:00'}], 'ofd_hard_limit': 262144, 'ofd_soft_limit': 1024, 'physical_cores': 20, 'platform_architecture': 'x86_64', 'platform_node': 'localhost.localdomain', 'platform_release': '4.18.0-348.7.1.el8_5.x86_64', 'platform_version': '#1 SMP Wed Dec 22 13:25:12 UTC 2021', 'total_memory': '125.52GB', 'used_memory': '7.96GB'}, 'job_type': 'homo_abinit', 'params_spec': {'abinit_K': {'value': 10}, 'compute_use_ssd': {'value': False}}, 'project_uid': 'P17', 'status': 'failed', 'uid': 'J200', 'version': 'v4.6.0'}
(base) [cryosparc_user@localhost ~]$ cryosparcm eventlog $csprojectid $csjobid | tail -n 40
[Sat, 22 Feb 2025 04:52:56 GMT] [CPU RAM used: 4089 MB] -- Class 1 -- lr: 0.10 eps: 247368961144398997838455705174016.00 step ratio : 0.0000 ESS R: 1.002 S: 11.994 Class Size: 0.0% (Average: 10.1%)
[Sat, 22 Feb 2025 04:52:56 GMT] [CPU RAM used: 4089 MB] -- Class 2 -- lr: 0.10 eps: 247368961144398997838455705174016.00 step ratio : 0.0000 ESS R: 1.003 S: 11.987 Class Size: 0.0% (Average: 9.7%)
[Sat, 22 Feb 2025 04:52:57 GMT] [CPU RAM used: 4089 MB] -- Class 3 -- lr: 0.10 eps: 247368961144398997838455705174016.00 step ratio : 0.0000 ESS R: 1.002 S: 11.991 Class Size: 0.0% (Average: 10.3%)
[Sat, 22 Feb 2025 04:52:57 GMT] [CPU RAM used: 4089 MB] -- Class 4 -- lr: 0.10 eps: 247368961144398997838455705174016.00 step ratio : 0.0000 ESS R: 1.001 S: 11.986 Class Size: 0.0% (Average: 10.1%)
[Sat, 22 Feb 2025 04:52:57 GMT] [CPU RAM used: 4089 MB] -- Class 5 -- lr: 0.10 eps: 247368961144398997838455705174016.00 step ratio : 0.0000 ESS R: 1.007 S: 11.998 Class Size: 0.0% (Average: 9.8%)
[Sat, 22 Feb 2025 04:52:57 GMT] [CPU RAM used: 4089 MB] -- Class 6 -- lr: 0.10 eps: 247368961144398997838455705174016.00 step ratio : nan ESS R: 1.001 S: 12.000 Class Size: 62.1% (Average: 10.1%)
[Sat, 22 Feb 2025 04:52:57 GMT] [CPU RAM used: 4089 MB] -- Class 7 -- lr: 0.10 eps: 247368961144398997838455705174016.00 step ratio : nan ESS R: 1.006 S: 12.000 Class Size: 32.7% (Average: 9.7%)
[Sat, 22 Feb 2025 04:52:57 GMT] [CPU RAM used: 4089 MB] -- Class 8 -- lr: 0.10 eps: 247368961144398997838455705174016.00 step ratio : 0.0000 ESS R: 1.002 S: 11.986 Class Size: 0.0% (Average: 10.1%)
[Sat, 22 Feb 2025 04:52:57 GMT] [CPU RAM used: 4089 MB] -- Class 9 -- lr: 0.10 eps: 247368961144398997838455705174016.00 step ratio : 0.0000 ESS R: 1.001 S: 11.999 Class Size: 5.3% (Average: 10.1%)
[Sat, 22 Feb 2025 04:53:03 GMT] [CPU RAM used: 4072 MB] Done iteration 02682 of 03069 in 14.865s. Total time 38061.1s. Est time remaining 5950.4s.
[Sat, 22 Feb 2025 04:53:03 GMT] [CPU RAM used: 4152 MB] ----------- Iteration 2683 (epoch 1.389). radwn 48.00 resolution 12.00A minisize 300 beta 0.00
[Sat, 22 Feb 2025 04:53:11 GMT] [CPU RAM used: 4150 MB] -- Class 0 -- lr: 0.10 eps: 331867415258755320856599144169472.00 step ratio : 0.0000 ESS R: 1.002 S: 11.994 Class Size: 0.0% (Average: 9.9%)
[Sat, 22 Feb 2025 04:53:11 GMT] [CPU RAM used: 4150 MB] -- Class 1 -- lr: 0.10 eps: 331867415258755320856599144169472.00 step ratio : nan ESS R: 1.005 S: 12.001 Class Size: 9.0% (Average: 10.1%)
[Sat, 22 Feb 2025 04:53:12 GMT] [CPU RAM used: 4150 MB] -- Class 2 -- lr: 0.10 eps: 331867415258755320856599144169472.00 step ratio : 0.0000 ESS R: 0.999 S: 11.988 Class Size: 0.0% (Average: 9.7%)
[Sat, 22 Feb 2025 04:53:12 GMT] [CPU RAM used: 4150 MB] -- Class 3 -- lr: 0.10 eps: 331867415258755320856599144169472.00 step ratio : 0.0000 ESS R: 1.000 S: 11.990 Class Size: 0.0% (Average: 10.3%)
[Sat, 22 Feb 2025 04:53:12 GMT] [CPU RAM used: 4150 MB] -- Class 4 -- lr: 0.10 eps: 331867415258755320856599144169472.00 step ratio : 0.0000 ESS R: 1.000 S: 11.990 Class Size: 0.0% (Average: 10.1%)
[Sat, 22 Feb 2025 04:53:12 GMT] [CPU RAM used: 4150 MB] -- Class 5 -- lr: 0.10 eps: 331867415258755320856599144169472.00 step ratio : 0.0000 ESS R: 1.007 S: 12.000 Class Size: 0.0% (Average: 9.8%)
[Sat, 22 Feb 2025 04:53:12 GMT] [CPU RAM used: 4150 MB] -- Class 6 -- lr: 0.10 eps: 331867415258755320856599144169472.00 step ratio : 0.0000 ESS R: 1.000 S: 11.994 Class Size: 0.0% (Average: 10.1%)
[Sat, 22 Feb 2025 04:53:12 GMT] [CPU RAM used: 4150 MB] -- Class 7 -- lr: 0.10 eps: 331867415258755320856599144169472.00 step ratio : 0.0000 ESS R: 1.000 S: 11.994 Class Size: 0.0% (Average: 9.7%)
[Sat, 22 Feb 2025 04:53:12 GMT] [CPU RAM used: 4150 MB] -- Class 8 -- lr: 0.10 eps: 331867415258755320856599144169472.00 step ratio : 0.0000 ESS R: 0.999 S: 11.988 Class Size: 0.0% (Average: 10.1%)
[Sat, 22 Feb 2025 04:53:12 GMT] [CPU RAM used: 4150 MB] -- Class 9 -- lr: 0.10 eps: 331867415258755320856599144169472.00 step ratio : nan ESS R: 1.001 S: 12.000 Class Size: 91.2% (Average: 10.2%)
[Sat, 22 Feb 2025 04:53:18 GMT] [CPU RAM used: 4141 MB] Done iteration 02683 of 03069 in 15.226s. Total time 38076.3s. Est time remaining 5927.6s.
[Sat, 22 Feb 2025 04:53:18 GMT] [CPU RAM used: 4198 MB] ----------- Iteration 2684 (epoch 1.389). radwn 48.00 resolution 12.00A minisize 300 beta 0.00
[Sat, 22 Feb 2025 04:53:26 GMT] [CPU RAM used: 4205 MB] -- Class 0 -- lr: 0.10 eps: 408168986694449104101698806218752.00 step ratio : 0.0000 ESS R: 1.000 S: 12.000 Class Size: 0.0% (Average: 9.9%)
[Sat, 22 Feb 2025 04:53:27 GMT] [CPU RAM used: 4205 MB] -- Class 1 -- lr: 0.10 eps: 408168986694449104101698806218752.00 step ratio : 0.0000 ESS R: 0.999 S: 11.994 Class Size: 0.0% (Average: 10.1%)
[Sat, 22 Feb 2025 04:53:27 GMT] [CPU RAM used: 4205 MB] -- Class 2 -- lr: 0.10 eps: 408168986694449104101698806218752.00 step ratio : 0.0000 ESS R: 1.002 S: 11.987 Class Size: 0.0% (Average: 9.7%)
[Sat, 22 Feb 2025 04:53:27 GMT] [CPU RAM used: 4205 MB] -- Class 3 -- lr: 0.10 eps: 408168986694449104101698806218752.00 step ratio : 0.0000 ESS R: 1.002 S: 11.998 Class Size: 0.0% (Average: 10.3%)
[Sat, 22 Feb 2025 04:53:27 GMT] [CPU RAM used: 4205 MB] -- Class 4 -- lr: 0.10 eps: 408168986694449104101698806218752.00 step ratio : 0.0000 ESS R: 1.000 S: 11.990 Class Size: 0.0% (Average: 10.0%)
[Sat, 22 Feb 2025 04:53:27 GMT] [CPU RAM used: 4205 MB] -- Class 5 -- lr: 0.10 eps: 408168986694449104101698806218752.00 step ratio : nan ESS R: 1.010 S: 12.000 Class Size: 100.2% (Average: 9.9%)
[Sat, 22 Feb 2025 04:53:27 GMT] [CPU RAM used: 4205 MB] -- Class 6 -- lr: 0.10 eps: 408168986694449104101698806218752.00 step ratio : 0.0000 ESS R: 1.002 S: 12.000 Class Size: 0.0% (Average: 10.1%)
[Sat, 22 Feb 2025 04:53:27 GMT] [CPU RAM used: 4205 MB] -- Class 7 -- lr: 0.10 eps: 408168986694449104101698806218752.00 step ratio : 0.0000 ESS R: 1.007 S: 11.999 Class Size: 0.0% (Average: 9.7%)
[Sat, 22 Feb 2025 04:53:27 GMT] [CPU RAM used: 4205 MB] -- Class 8 -- lr: 0.10 eps: 408168986694449104101698806218752.00 step ratio : 0.0000 ESS R: 1.002 S: 11.988 Class Size: 0.0% (Average: 10.0%)
[Sat, 22 Feb 2025 04:53:28 GMT] [CPU RAM used: 4205 MB] -- Class 9 -- lr: 0.10 eps: 408168986694449104101698806218752.00 step ratio : 0.0000 ESS R: 1.002 S: 11.989 Class Size: 0.0% (Average: 10.2%)
[Sat, 22 Feb 2025 04:53:33 GMT] [CPU RAM used: 4209 MB] Done iteration 02684 of 03069 in 14.927s. Total time 38091.3s. Est time remaining 5894.2s.
[Sat, 22 Feb 2025 04:53:34 GMT] [CPU RAM used: 3210 MB] Traceback (most recent call last):
File "cryosparc_master/cryosparc_compute/run.py", line 116, in cryosparc_master.cryosparc_compute.run.main
File "cryosparc_master/cryosparc_compute/jobs/abinit/run.py", line 301, in cryosparc_master.cryosparc_compute.jobs.abinit.run.run_homo_abinit
File "/home/cryosparc_user/cryosparc/cryosparc_worker/cryosparc_compute/noise_model.py", line 119, in get_noise_estimate
assert n.all(n.isfinite(ret))
AssertionError
(base) [cryosparc_user@localhost ~]$