Alternative MongoDB repair

Install, Configure and Update
1

Hello,

A power failure caused the computer I’m running cryosparc on to crash so crysparc stopped running, I restarted it but noticed that the 3 attempts to find the database all failed and that the program wouldn’t run. I then tried repairing it using:
./cryosparc_master/deps_bundle/external/mongodb/bin/mongod --dbpath /home/cryosparc/cryosparc_database --repair but it doesn’t come back up. This is the backtrace:

----- BEGIN BACKTRACE -----
{"backtrace":[{"b":"55A4043DF000","o":"22A5F21","s":"_ZN5mongo15printStackTraceERSo"},{"b":"55A4043DF000","o":"22A5139"},{"b":"55A4043DF000","o":"22A561D"},{"b":"7FE4E38D2000","o":"42520"},{"b":"7FE4E38D2000","o":"96A7C","s":"pthread_kill"},{"b":"7FE4E38D2000","o":"42476","s":"raise"},{"b":"7FE4E38D2000","o":"287F3","s":"abort"},{"b":"55A4043DF000","o":"989DEC","s":"_ZN5mongo32fassertFailedNoTraceWithLocationEiPKcj"},{"b":"55A4043DF000","o":"A64D76"},{"b":"55A4043DF000","o":"AD6AD1"},{"b":"55A4043DF000","o":"926A94","s":"__wt_err_func"},{"b":"55A4043DF000","o":"926EB4","s":"__wt_panic"},{"b":"55A4043DF000","o":"B8D3B5","s":"__wt_block_read_off"},{"b":"55A4043DF000","o":"BABD6F","s":"__wt_block_extlist_read"},{"b":"55A4043DF000","o":"BAC2CB","s":"__wt_block_extlist_read_avail"},{"b":"55A4043DF000","o":"BA86B5","s":"__wt_block_checkpoint_load"},{"b":"55A4043DF000","o":"B8CA57"},{"b":"55A4043DF000","o":"AFED5E","s":"__wt_btree_open"},{"b":"55A4043DF000","o":"A783F2","s":"__wt_conn_dhandle_open"},{"b":"55A4043DF000","o":"AD53BD","s":"__wt_session_get_dhandle"},{"b":"55A4043DF000","o":"AD599D","s":"__wt_session_get_dhandle"},{"b":"55A4043DF000","o":"AD5C2C","s":"__wt_session_get_btree_ckpt"},{"b":"55A4043DF000","o":"B47E52","s":"__wt_curfile_open"},{"b":"55A4043DF000","o":"AD0EC8"},{"b":"55A4043DF000","o":"A9797E","s":"__wt_metadata_cursor_open"},{"b":"55A4043DF000","o":"A97A5B","s":"__wt_metadata_cursor"},{"b":"55A4043DF000","o":"A76C8A","s":"wiredtiger_open"},{"b":"55A4043DF000","o":"A43CF6","s":"_ZN5mongo18WiredTigerKVEngineC2ERKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEES8_PNS_11ClockSourceES8_mmbbbb"},{"b":"55A4043DF000","o":"A25AEC"},{"b":"55A4043DF000","o":"C35FB6","s":"_ZN5mongo20ServiceContextMongoD29initializeGlobalStorageEngineEv"},{"b":"55A4043DF000","o":"A025B8"},{"b":"55A4043DF000","o":"A0563C","s":"_ZN5mongo11mongoDbMainEiPPcS1_"},{"b":"55A4043DF000","o":"98BBC9","s":"main"},{"b":"7FE4E38D2000","o":"29D90"},{"b":"7FE4E38D2000","o":"29E40","s":"__libc_start_main"},{"b":"55A4043DF000","o":"9ED741"}],"processInfo":{ "mongodbVersion" : "3.6.23", "gitVersion" : "d352e6a4764659e0d0350ce77279de3c1f243e5c", "compiledModules" : [], "uname" : { "sysname" : "Linux", "release" : "5.15.0-76-generic", "version" : "#83-Ubuntu SMP Thu Jun 15 19:16:32 UTC 2023", "machine" : "x86_64" }, "somap" : [ { "b" : "55A4043DF000", "elfType" : 3, "buildId" : "B0818C001F2B63D4533D208F68F08AE2A599CA9E" }, { "b" : "7FFF69300000", "path" : "linux-vdso.so.1", "elfType" : 3, "buildId" : "BCBFC809B031BF916E78FB3EB18509D1EAF3BB3D" }, { "b" : "7FE4E3C12000", "path" : "/lib/x86_64-linux-gnu/libresolv.so.2", "elfType" : 3, "buildId" : "7FD7253C61AA6FCE2B7E13851C15AFA14A5AB160" }, { "b" : "7FE4E3C0D000", "path" : "/lib/x86_64-linux-gnu/librt.so.1", "elfType" : 3, "buildId" : "1030BE4690F8AA93B63ACD9180B3037D99A974BF" }, { "b" : "7FE4E3C08000", "path" : "/lib/x86_64-linux-gnu/libdl.so.2", "elfType" : 3, "buildId" : "952123C3BDCE7C3370553717942B4878D1FAD797" }, { "b" : "7FE4E3B21000", "path" : "/lib/x86_64-linux-gnu/libm.so.6", "elfType" : 3, "buildId" : "27E82301DBA6C3F644404D504E1BB1C97894B433" }, { "b" : "7FE4E3B01000", "path" : "/lib/x86_64-linux-gnu/libgcc_s.so.1", "elfType" : 3, "buildId" : "09C4935B79388431A1248F6A98E00D7DC81B8513" }, { "b" : "7FE4E3AFA000", "path" : "/lib/x86_64-linux-gnu/libpthread.so.0", "elfType" : 3, "buildId" : "E62798B68557ABB4BC5548ABA2640CD5AB948F36" }, { "b" : "7FE4E38D2000", "path" : "/lib/x86_64-linux-gnu/libc.so.6", "elfType" : 3, "buildId" : "69389D485A9793DBE873F0EA2C93E02EFAA9AA3D" }, { "b" : "7FE4E3C41000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "61EF896A699BB1C2E4E231642B2E1688B2F1A61E" } ] }}
 mongod(_ZN5mongo15printStackTraceERSo+0x41) [0x55a406684f21]
 mongod(+0x22A5139) [0x55a406684139]
 mongod(+0x22A561D) [0x55a40668461d]
 libc.so.6(+0x42520) [0x7fe4e3914520]
 libc.so.6(pthread_kill+0x12C) [0x7fe4e3968a7c]
 libc.so.6(raise+0x16) [0x7fe4e3914476]
 libc.so.6(abort+0xD3) [0x7fe4e38fa7f3]
 mongod(_ZN5mongo32fassertFailedNoTraceWithLocationEiPKcj+0x0) [0x55a404d68dec]
 mongod(+0xA64D76) [0x55a404e43d76]
 mongod(+0xAD6AD1) [0x55a404eb5ad1]
 mongod(__wt_err_func+0x90) [0x55a404d05a94]
 mongod(__wt_panic+0x3F) [0x55a404d05eb4]
 mongod(__wt_block_read_off+0x585) [0x55a404f6c3b5]
 mongod(__wt_block_extlist_read+0x8F) [0x55a404f8ad6f]
 mongod(__wt_block_extlist_read_avail+0x2B) [0x55a404f8b2cb]
 mongod(__wt_block_checkpoint_load+0x275) [0x55a404f876b5]
 mongod(+0xB8CA57) [0x55a404f6ba57]
 mongod(__wt_btree_open+0xD7E) [0x55a404eddd5e]
 mongod(__wt_conn_dhandle_open+0x352) [0x55a404e573f2]
 mongod(__wt_session_get_dhandle+0xED) [0x55a404eb43bd]
 mongod(__wt_session_get_dhandle+0x6CD) [0x55a404eb499d]
 mongod(__wt_session_get_btree_ckpt+0x14C) [0x55a404eb4c2c]
 mongod(__wt_curfile_open+0x52) [0x55a404f26e52]
 mongod(+0xAD0EC8) [0x55a404eafec8]
 mongod(__wt_metadata_cursor_open+0x6E) [0x55a404e7697e]
 mongod(__wt_metadata_cursor+0x4B) [0x55a404e76a5b]
 mongod(wiredtiger_open+0x1BBA) [0x55a404e55c8a]
 mongod(_ZN5mongo18WiredTigerKVEngineC2ERKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEES8_PNS_11ClockSourceES8_mmbbbb+0x8D6) [0x55a404e22cf6]
 mongod(+0xA25AEC) [0x55a404e04aec]
 mongod(_ZN5mongo20ServiceContextMongoD29initializeGlobalStorageEngineEv+0x266) [0x55a405014fb6]
 mongod(+0xA025B8) [0x55a404de15b8]
 mongod(_ZN5mongo11mongoDbMainEiPPcS1_+0x26C) [0x55a404de463c]
 mongod(main+0x9) [0x55a404d6abc9]
 libc.so.6(+0x29D90) [0x7fe4e38fbd90]
 libc.so.6(__libc_start_main+0x80) [0x7fe4e38fbe40]
 mongod(+0x9ED741) [0x55a404dcc741]
-----  END BACKTRACE  -----
Aborted (core dumped)

Is there another way of repairing mongodb with my specific directory?
I’ve tried running mongod using just the database directory but it fails.
I’ve checked that there aren’t any other instances of cryosparc running and I stopped cryosparc before the repair.

Thank you

2

@ldlamini If there any error messages immediately above the backtrace, please can you post them.

3

Hi,
There aren’t any.
The backtrace is what the repair command returns.

4

To add to ldlamini’s question:
The last backup of mongoDB was made about a month ago. All of the cryosparc output is available.
Is it possible to rebuild mongoDB from the cryosparc outputs? Alternatively are there shortcuts that we can take in reprocessing using the outputs that we have available?

5

@ldlamini If error messages in the database log
cryosparcm log database
confirm corruption, I recommend:

  1. If possible, harden the server against power failures:
    • “uninterruptible” power
    • redundant power supplies
  2. As @btsewell suggested, rebuild a new database from your project directories. The procedure is described in a related post:

[edited for spelling 2024-04-12]

6

Hi @wtempel

Thank you for your response.
I have detached the previous project and attached a new one. The jobs that were in the previous project were attached, the problem is, the attached jobs account for only about 20% of all the processing I’ve done. The rest aren’t in the new project.
I have the database with all jobs saved so I copied one from CS-xxxx into CS-xxxxx/imports and imported the job into the new project. The import was successful and the directory contains all the data but the job data doesn’t appear on the web interface. I see that the import function doesn’t copy all data files for large data but the data are “maintained as symlinks” to the original files.
How then would I continue processing from where I stopped if I can’t see the job output from the parent jobs on the interface so that I can link it to the next job, or link output from earlier jobs to later jobs?
Would I have to go into each job and copy the specific output data I need?
I’m trying not to have to re-do all the jobs.

Thanks!

7

Which element of the CryoSPARC graphical or command line interface did you use for detachment?

Please describe the process you used for attachment.

Were the missing jobs part of the same project or other project(s), which would have to be attached separately?

With database, do you mean project directory (or directories)?

Job import requires a prior export of the job; a mere copy of the job directory is not expected to work for the purposes of the Import Job function.

8

“Which element of the CryoSPARC graphical or command line interface did you use for detachment?”
I used the detach button on the project actions drop-down menu on the cryosparc web interface.

“Please describe the process you used for attachment.”
I attached a new project using the “attach project” button in the Projects menu. A new project was created with the same name as the previous one and the jobs appearing in the old project appeared in the new one.

“Were the missing jobs part of the same project or other project(s), which would have to be attached separately?”
They were part of the same project.

“With database, do you mean project directory (or directories)?”
I mean project directory, sorry.

These are the instructions that I followed

9

The fact that you were able to navigate to the project to detach it suggests that the database may not be corrupt and that the recommendation to

may not apply. That recommendation was meant for the case that the database was corrupt.
Please can you post the output of these commands:

cryosparcm status | grep -v LICENSE_ID
ps -eopid,ppid,start,cmd | grep -e cryosparc -e mongo

Were the jobs now missing from the project still visible immediately before you detached the project?
You may also want to inspect the command_core log for errors that may have occurred during the re-attachment of the project

csr filterlog -f import_jobs command_core
10

Hi @wtempel

“cryosparcm status | grep -v LICENSE_ID” returns

CryoSPARC System master node installed at
/home/cryosparc/cryosparc_master
Current cryoSPARC version: v4.2.1
----------------------------------------------------------------------------

CryoSPARC process status:

app                              RUNNING   pid 673021, uptime 1 day, 1:35:31
app_api                          RUNNING   pid 673260, uptime 1 day, 1:35:29
app_api_dev                      STOPPED   Not started
app_legacy                       STOPPED   Not started
app_legacy_dev                   STOPPED   Not started
command_core                     RUNNING   pid 672335, uptime 1 day, 1:35:46
command_rtp                      RUNNING   pid 672982, uptime 1 day, 1:35:36
command_vis                      RUNNING   pid 672687, uptime 1 day, 1:35:37
database                         RUNNING   pid 672168, uptime 1 day, 1:35:50

----------------------------------------------------------------------------
License is valid
----------------------------------------------------------------------------

global config variables:
export CRYOSPARC_MASTER_HOSTNAME="athena.uct.ac.za"
export CRYOSPARC_DB_PATH="/data/cryosparc_database"
export CRYOSPARC_BASE_PORT=39000
export CRYOSPARC_DEVELOP=false
export CRYOSPARC_INSECURE=false
export CRYOSPARC_CLICK_WRAP=true

" ps -eopid,ppid,start,cmd | grep -e cryosparc -e mongo"

cryosparc@athena:~$ ps -eopid,ppid,start,cmd | grep -e cryosparc -e mongo
 671759       1   Jul 25 python /home/cryosparc/cryosparc_master/deps/anaconda/envs/cryosparc_master_env/bin/supervisord -c /home/cryosparc/cryosparc_master/supervisord.conf
 672168  671759   Jul 25 mongod --auth --dbpath /data/cryosparc_database --port 39001 --oplogSize 64 --replSet meteor --nojournal --wiredTigerCacheSizeGB 4 --bind_ip_all
 672335  671759   Jul 25 python -c import cryosparc_command.command_core as serv; serv.start(port=39002)
 672687  671759   Jul 25 python -c import cryosparc_command.command_vis as serv; serv.start(port=39003)
 672982  671759   Jul 25 python -c import cryosparc_command.command_rtp as serv; serv.start(port=39005)
 673260  671759   Jul 25 /home/cryosparc/cryosparc_master/cryosparc_app/api/nodejs/bin/node ./bundle/main.js
1006045 1002707 15:57:49 ssh cryosparc@athena
1006046    8187 15:57:49 sshd: cryosparc [priv]
1006212 1006046 15:57:52 sshd: cryosparc@pts/1
1008082 1006213 15:59:09 grep --color=auto -e cryosparc -e mongo

“Were the jobs now missing from the project still visible immediately before you detached the project?”
No, they were not, they “disappeared” from the web interface after I attempted to repair the database.
I don’t think the detaching and attaching changed anything, all the jobs that were there before the attachment are in the new project.

It might be easier and quicker to re-do everything and back it up more often.

11

That depends on how much time would it take to “re-do everything”, and what information is still present in your project directory.
What outputs do you get when open a Linux shell as user cryosparc and

cd /your/cryosparc/project_directory
cat job_manifest.json | grep "J[0-9]" | wc -l
find J[0-9]* -name job.json | wc -l
12

It took me about six weeks to get to the refined model without having Fourier cropped.
The cat command returns 55, it’s got all the remaining jobs and a few lines.
“find J[0-9]* -name job.json | wc -l” returns 358737

13

Can you see 55 jobs in the GUI for this project? How many workspaces did you have inside the project before the crash? Can you see all those workspaces?

This command may not have worked as intended. It was intended to find the project’s job directories with their respective job.json files inside.
Please can you post the output of this command, executed inside the project directory:
ls -F
If the output is very longs, please post 20 lines of the ouptut.