Hi all
After the upgrade to 2.11, some of our users are reporting that cloning a job or using an output from an existing job results in the job failing immediately.
The error is the same:
Traceback (most recent call last): File "cryosparc2_worker/cryosparc2_compute/run.py", line 78, in cryosparc2_compute.run.main File "cryosparc2_worker/cryosparc2_compute/jobs/class2D/run.py", line 53, in cryosparc2_compute.jobs.class2D.run.run_class_2D File "cryosparc2_compute/particles.py", line 30, in init self.from_dataset(d) # copies in all data File "cryosparc2_compute/dataset.py", line 362, in from_dataset self.data = other.data.copy() AttributeError: 'NoneType' object has no attribute 'data'
And the jobs usually fail immediately.
I have attached the metadata output from one of the failed jobs below.
It seems that the DB upgrade might have messed things up a little bit?
Any suggestion on how to fix this is welcome!
{
"_id": {
"_str": "5d9546ebfa9310809bd58307"
},
"children": [],
"created_at": "2019-10-03T00:55:07.859Z",
"created_by_job_uid": null,
"created_by_user_id": "5bdb963fbec69958f764ae0f",
"interactive": false,
"job_type": "class_2D",
"launched_at": "2019-10-03T01:07:44.456Z",
"parents": [
"J177"
],
"project_uid": "P35",
"queued_at": "2019-10-03T00:57:00.543Z",
"started_at": "2019-10-03T01:07:45.017Z",
"status": "failed",
"title": "New Job J213",
"uid": "J213",
"waiting_at": null,
"workspace_uids": [
"W2"
],
"cloned_from": "J209",
"deleted": false,
"type": "class_2D",
"ui_tile_height": 1,
"ui_tile_images": [],
"ui_tile_width": 3,
"completed_at": null,
"description": "Testing - Jaf",
"failed_at": "2019-10-03T01:07:51.612Z",
"job_dir_size": 0,
"killed_at": null,
"is_experiment": false,
"experiment_worker_path": null,
"output_group_images": {},
"queue_status": null,
"interactive_port": null,
"params_spec": {
"class2D_num_full_iter_batch": {
"value": 25
},
"class2D_K": {
"value": 30
},
"class2D_force_max": {
"value": false
},
"class2D_num_full_iter_batchsize_per_class": {
"value": 700
},
"class2D_max_res": {
"value": 10
}
},
"input_slot_groups": [
{
"name": "particles",
"title": "Particle stacks",
"repeat_allowed": false,
"connections": [
{
"slots": [
{
"job_uid": "J177",
"group_name": "particles",
"version": "F",
"result_name": "blob",
"slot_name": "blob",
"result_type": "particle.blob"
},
{
"job_uid": "J177",
"group_name": "particles",
"version": "F",
"result_name": "ctf",
"slot_name": "ctf",
"result_type": "particle.ctf"
},
{
"job_uid": "J177",
"group_name": "particles",
"version": "F",
"result_name": "location",
"slot_name": null,
"result_type": "particle.location"
},
{
"job_uid": "J177",
"group_name": "particles",
"version": "F",
"result_name": "pick_stats",
"slot_name": null,
"result_type": "particle.pick_stats"
}
],
"job_uid": "J177",
"group_name": "particles"
}
],
"count_max": null,
"slots": [
{
"type": "particle.blob",
"title": "Particle data blobs",
"optional": false,
"description": "",
"name": "blob"
},
{
"type": "particle.ctf",
"title": "Particle ctf parameters",
"optional": false,
"description": "",
"name": "ctf"
}
],
"type": "particle",
"count_min": 1,
"description": "Particle stacks to use. Multiple stacks will be concatenated."
}
],
"params_base": {
"class2D_window": {
"title": "Use circular mask on 2D classes",
"section": "class_2D",
"value": true,
"hidden": false,
"type": "boolean",
"order": 6,
"advanced": true,
"desc": "Whether or not to apply a circular window to the 2D classes during classification. This ensures that each 2D class has no density outside the circular window. By default, the window is a circle that only masks out the corners of the 2D classes."
},
"class2D_recenter_thresh": {
"title": "Re-center mask threshold",
"section": "class_2D",
"value": 0.2,
"hidden": false,
"type": "number",
"order": 10,
"advanced": true,
"desc": "2D classes are recentered by computing the center-of-mass (COM) of pixels that are above this threshold value. The threshold is relative to the maximum density value in the reference, so 0.2 means pixels with greater than 20%% of the maximum density."
},
"class2D_max_res": {
"title": "Maximum resolution (A)",
"section": "class_2D",
"value": 6,
"hidden": false,
"type": "number",
"order": 4,
"advanced": false,
"desc": "The maximum resolution in Angstroms to consider when aligning and reconstructing 2D classes. This setting controls the box size that is used internally, and higher resolutions can slow down processing."
},
"class2D_num_full_iter_batch": {
"title": "Number of online-EM iterations",
"section": "class_2D",
"value": 20,
"hidden": false,
"type": "number",
"order": 16,
"advanced": true,
"desc": "The total number of iterations of online-EM to perform. Typically 20 is enough, but for small or low SNR particles, or when classifying subsets that have few distinct views, a larger number like 40 can help."
},
"class2D_K": {
"title": "Number of 2D classes",
"section": "class_2D",
"value": 50,
"hidden": false,
"type": "number",
"order": 3,
"advanced": false,
"desc": "The number of 2D classes into which to sort the dataset. Runtime is approximately linear in the number of classes. Often, sorting the same dataset into different numbers of classes (using multiple jobs) can be helpful in finding junk particles or rare views."
},
"class2D_recenter_binary": {
"title": "Re-center mask binary",
"section": "class_2D",
"value": false,
"hidden": false,
"type": "boolean",
"order": 11,
"advanced": true,
"desc": "If True, compute the COM for re-centering by equally weighting every pixel that was above the threshold. If False, weight every pixel by its greyscale density value."
},
"class2D_use_frc_reg": {
"title": "Use FRC based regularizer",
"section": "class_2D",
"value": true,
"hidden": false,
"type": "boolean",
"order": 24,
"advanced": true,
"desc": "Use an FRC based regularizer to avoid overfitting during classification."
},
"class2D_ctf_phase_flip_only": {
"title": "CTF flip phases only",
"section": "class_2D",
"value": false,
"hidden": false,
"type": "boolean",
"order": 13,
"advanced": true,
"desc": "Treat the CTF by flipping phases only, rather that correctly accounting for amplitude and phase. Not recommended."
},
"class2D_window_outer_A": {
"title": "Circular mask diameter outer (A)",
"section": "class_2D",
"value": null,
"hidden": false,
"type": "number",
"order": 8,
"advanced": true,
"desc": "The outer diameter (in Angstroms) of the window. If None, outer diameter is 20 percent larger than inner diameter. The window mask transitions smoothly between inner and outer diameters."
},
"class2D_ignore_dc": {
"title": "Ignore DC from image data",
"section": "class_2D",
"value": true,
"hidden": false,
"type": "boolean",
"order": 20,
"advanced": true,
"desc": "Whether or not to ignore the DC component of particle images (i.e. float the images)"
},
"class2D_num_full_iter": {
"title": "Number of final full iterations",
"section": "class_2D",
"value": 1,
"hidden": false,
"type": "number",
"order": 15,
"advanced": true,
"desc": "The number of final full passes through the dataset at the end of classification. Usually only one full pass is needed."
},
"class2D_sigma_use_white": {
"title": "Use white noise model",
"section": "class_2D",
"value": false,
"hidden": false,
"type": "boolean",
"order": 28,
"advanced": true,
"desc": "Force the use of a white noise model."
},
"class2D_clamp": {
"title": "Use clamp-solvent to solve 2D classes",
"section": "class_2D",
"value": false,
"hidden": false,
"type": "boolean",
"order": 23,
"advanced": true,
"desc": "Use solvent-clamping to solve 2D references. This, coupled with non-negativity, can drastically reduce background noise and often provides cleaner 2D references."
},
"class2D_window_inner_A": {
"title": "Circular mask diameter (A)",
"section": "class_2D",
"value": null,
"hidden": false,
"type": "number",
"order": 7,
"advanced": false,
"desc": "The inner diameter (in Angstroms) of the window that is applied to 2D classes during classification. If None, the window only masks out the corners of each 2D class."
},
"class2D_random_perturb_rs": {
"title": "Randomly perturb poses radians",
"section": "class_2D",
"value": null,
"hidden": false,
"type": "number",
"order": 14,
"advanced": true,
"desc": "Randomly perturb poses after alignment to within this number of radians. This can sometimes help with classification by injecting more noise and stopping any potential overfitting, but usually results in lower resolution class averages."
},
"prepare_window_inner_radius": {
"title": "Window inner radius",
"section": "prepare",
"value": 0.85,
"hidden": false,
"type": "number",
"order": 1,
"advanced": true,
"desc": ""
},
"class2D_sigma_init_factor": {
"title": "Initial classification uncertainty factor",
"section": "class_2D",
"value": 2,
"hidden": false,
"type": "number",
"order": 5,
"advanced": false,
"desc": "This factor (a number greater than 1) controls the initial search for 2D references. A value of 1.0 indicates that the search should quickly become certain about classes and assignments, leading to the algorithm finding more \"junk\" classes. A value larger than 1.0 (usually between 2 and 10) causes the algorithm to remain uncertain about classes and assignments for more iterations, resulting in more diversity of \"good\" classes."
},
"random_seed": {
"title": "Random seed",
"section": "random",
"value": null,
"hidden": false,
"type": "number",
"order": 30,
"advanced": true,
"desc": "Set to null to auto generate."
},
"class2D_sigma_num_anneal_iters": {
"title": "Number of iteration to anneal sigma",
"section": "class_2D",
"value": 15,
"hidden": false,
"type": "number",
"order": 27,
"advanced": true,
"desc": "Number of iterations over which to anneal noise model. Not recommended to change."
},
"prepare_window_dataset": {
"title": "Window dataset (real-space)",
"section": "prepare",
"value": true,
"hidden": false,
"type": "boolean",
"order": 0,
"advanced": false,
"desc": ""
},
"class2D_recenter": {
"title": "Re-center 2D classes",
"section": "class_2D",
"value": true,
"hidden": false,
"type": "boolean",
"order": 9,
"advanced": false,
"desc": "Whether or not to re-center 2D class references at every iteration to avoid drift of density away from the center of the box. This option is often important to keep classes centered and avoid artefacts near the edges of the box."
},
"class2D_init_scale": {
"title": "2D initial scale",
"section": "class_2D",
"value": 1,
"hidden": false,
"type": "number",
"order": 18,
"advanced": true,
"desc": "Initial scale of random starting references. Not recommended to change."
},
"class2D_force_max": {
"title": "Force Max over poses/shifts",
"section": "class_2D",
"value": true,
"hidden": false,
"type": "boolean",
"order": 12,
"advanced": true,
"desc": "If True, maximize over poses and shifts when aligning particles to references. If False, marginalize over poses and shifts to account for alignment uncertainty. This is generally not necessary, but can provide better results with very small or low SNR particles."
},
"class2D_num_full_iter_batchsize_per_class": {
"title": "Batchsize per class",
"section": "class_2D",
"value": 100,
"hidden": false,
"type": "number",
"order": 17,
"advanced": true,
"desc": "The number of particles per class to use during each iteration of online-EM. For small or low SNR particles, this can be increased to 200."
},
"class2D_min_over_scale": {
"title": "Min over scale after first iteration",
"section": "class_2D",
"value": false,
"hidden": false,
"type": "boolean",
"order": 21,
"advanced": true,
"desc": "Whether or not to minimize over per-particle scale factors, after the first iteration. This can help if ice thicknesses varied greatly during data collection."
},
"compute_num_gpus": {
"title": "Number of GPUs to parallelize",
"section": "compute_settings",
"value": 1,
"hidden": false,
"type": "number",
"order": 32,
"advanced": false,
"desc": "Number of GPUs to use during classification"
},
"class2D_nonneg": {
"title": "Enforce non-negativity",
"section": "class_2D",
"value": false,
"hidden": false,
"type": "boolean",
"order": 22,
"advanced": true,
"desc": "Whether or not to enforce non-negativity of 2D references during optimization. This can help to avoid \"streaking\" artefacts."
},
"class2D_use_frc_reg_full": {
"title": "Use full FRC",
"section": "class_2D",
"value": true,
"hidden": false,
"type": "boolean",
"order": 25,
"advanced": true,
"desc": ""
},
"intermediate_plots": {
"title": "Show plots from intermediate steps",
"section": "class_2D",
"value": true,
"hidden": false,
"type": "boolean",
"order": 29,
"advanced": false,
"desc": "Whether or not to create and show intermediate plots before the last iteration. This can be turned off to optimize for speed."
},
"class2D_zp_factor": {
"title": "2D zeropad factor",
"section": "class_2D",
"value": 2,
"hidden": false,
"type": "number",
"order": 19,
"advanced": true,
"desc": "Zeropadding factor. For very large box particles, this can be reduced to speed up computation and reduce memory requirements."
},
"compute_use_ssd": {
"title": "Cache particle images on SSD",
"section": "compute_settings",
"value": true,
"hidden": false,
"type": "boolean",
"order": 31,
"advanced": false,
"desc": "Whether or not to copy particle images to the local SSD before running. The cache is persistent, so after caching once, particles should be available for subsequent jobs that require the same data. Not using an SSD can dramatically slow down processing."
},
"prepare_window_outer_radius": {
"title": "Window outer radius",
"section": "prepare",
"value": 0.99,
"hidden": false,
"type": "number",
"order": 2,
"advanced": true,
"desc": ""
},
"class2D_sigma_init_iter": {
"title": "Iteration to start annealing sigma",
"section": "class_2D",
"value": 2,
"hidden": false,
"type": "number",
"order": 26,
"advanced": true,
"desc": "Iteration at which noise model should be annealed. Not recommended to change."
}
},
"resources_allocated": {
"lane": "default",
"target": {
"lane": "default",
"worker_bin_path": "/mnt/cryosparc_home/cryosparc/worker/bin/cryosparcw",
"cache_reserve_mb": 10000,
"title": "Worker node m3h015.massive.org.au",
"resource_slots": {
"GPU": [
0,
1
],
"RAM": [
0,
1,
2,
3,
4,
5,
6,
7,
8,
9,
10,
11,
12,
13,
14,
15,
16,
17,
18,
19,
20,
21,
22,
23,
24,
25,
26,
27,
28,
29
],
"CPU": [
0,
1,
2,
3,
4,
5,
6,
7,
8,
9,
10,
11,
12,
13,
14,
15,
16,
17,
18,
19,
20,
21,
22,
23,
24,
25,
26,
27
]
},
"hostname": "m3h015.massive.org.au",
"cache_path": "/mnt/ssd",
"cache_quota_mb": null,
"resource_fixed": {
"SSD": true
},
"desc": null,
"type": "node",
"ssh_str": "cryosparc@m3h015.massive.org.au",
"name": "m3h015.massive.org.au"
},
"license": true,
"hostname": "m3h015.massive.org.au",
"slots": {
"GPU": [
1
],
"RAM": [
1,
4,
5
],
"CPU": [
2,
3
]
},
"fixed": {
"SSD": true
},
"lane_type": "node"
},
"job_dir": "J213",
"errors_run": [
{
"message": "'NoneType' object has no attribute 'data'",
"warning": false
}
],
"queued_to_lane": "default",
"run_on_master_direct": false,
"version": "v2.11.0",
"output_result_groups": [
{
"num_items": 0,
"uid": "J213-G0",
"title": "All particles",
"passthrough": "particles",
"description": "All particles that were processed, including alignments across all classes.",
"contains": [
{
"passthrough": false,
"type": "particle.blob",
"uid": "J213-R0",
"name": "blob",
"group_name": "particles"
},
{
"passthrough": false,
"type": "particle.ctf",
"uid": "J213-R1",
"name": "ctf",
"group_name": "particles"
},
{
"passthrough": false,
"type": "particle.alignments2D",
"uid": "J213-R2",
"name": "alignments2D",
"group_name": "particles"
},
{
"passthrough": true,
"type": "particle.location",
"uid": "J213-R3",
"name": "location",
"group_name": "particles"
},
{
"passthrough": true,
"type": "particle.pick_stats",
"uid": "J213-R4",
"name": "pick_stats",
"group_name": "particles"
}
],
"type": "particle",
"name": "particles"
},
{
"num_items": 0,
"uid": "J213-G1",
"title": "2D class averages",
"passthrough": false,
"description": "2D class average templates",
"contains": [
{
"passthrough": false,
"type": "template.blob",
"uid": "J213-R5",
"name": "blob",
"group_name": "class_averages"
}
],
"type": "template",
"name": "class_averages"
}
],
"run_as_user": null,
"errors_build_params": {},
"PID_main": 25863,
"errors_build_inputs": {},
"resources_needed": {
"slots": {
"GPU": 1,
"RAM": 3,
"CPU": 2
},
"fixed": {
"SSD": true
},
"license": true
},
"PID_workers": [],
"queue_message": null,
"params_secs": {
"class_2D": {
"title": "2D Classification",
"order": 1,
"desc": ""
},
"random": {
"title": "Random Seeds",
"order": 2,
"desc": ""
},
"compute_settings": {
"title": "Compute settings",
"order": 3,
"desc": ""
},
"prepare": {
"title": "Particle preprocessing",
"order": 0,
"desc": ""
}
},
"queue_index": null,
"output_results": [
{
"num_items": [],
"uid": "J213-R0",
"passthrough": false,
"min_fields": [
[
"path",
"O"
],
[
"idx",
"u4"
],
[
"shape",
"2u4"
],
[
"psize_A",
"f4"
],
[
"sign",
"f4"
]
],
"group_name": "particles",
"metafiles": [],
"description": "Particle raw data",
"name": "blob",
"title": "Particle data",
"versions": [],
"type": "particle.blob"
},
{
"num_items": [],
"uid": "J213-R1",
"passthrough": false,
"min_fields": [
[
"type",
"O"
],
[
"exp_group_id",
"u4"
],
[
"accel_kv",
"f4"
],
[
"cs_mm",
"f4"
],
[
"amp_contrast",
"f4"
],
[
"df1_A",
"f4"
],
[
"df2_A",
"f4"
],
[
"df_angle_rad",
"f4"
],
[
"phase_shift_rad",
"f4"
],
[
"scale",
"f4"
],
[
"scale_const",
"f4"
]
],
"group_name": "particles",
"metafiles": [],
"description": "Particle CTF parameters",
"name": "ctf",
"title": "Particle CTF parameters",
"versions": [],
"type": "particle.ctf"
},
{
"num_items": [],
"uid": "J213-R2",
"passthrough": false,
"min_fields": [
[
"split",
"u4"
],
[
"shift",
"2f4"
],
[
"pose",
"f4"
],
[
"psize_A",
"f4"
],
[
"error",
"f4"
],
[
"error_min",
"f4"
],
[
"resid_pow",
"f4"
],
[
"slice_pow",
"f4"
],
[
"image_pow",
"f4"
],
[
"cross_cor",
"f4"
],
[
"alpha",
"f4"
],
[
"weight",
"f4"
],
[
"pose_ess",
"f4"
],
[
"shift_ess",
"f4"
],
[
"class_posterior",
"f4"
],
[
"class",
"u4"
],
[
"class_ess",
"f4"
]
],
"group_name": "particles",
"metafiles": [],
"description": "2D alignments including pose, shift, stats, classification scores, etc.",
"name": "alignments2D",
"title": "Alignments for best class",
"versions": [],
"type": "particle.alignments2D"
},
{
"num_items": [],
"uid": "J213-R3",
"passthrough": true,
"min_fields": [
[
"micrograph_uid",
"u8"
],
[
"exp_group_id",
"u4"
],
[
"micrograph_path",
"O"
],
[
"micrograph_shape",
"2u4"
],
[
"center_x_frac",
"f4"
],
[
"center_y_frac",
"f4"
]
],
"group_name": "particles",
"metafiles": [],
"description": "Passthrough from input particles.location (result_name)",
"name": "location",
"title": "Passthrough location",
"versions": [],
"type": "particle.location"
},
{
"num_items": [],
"uid": "J213-R4",
"passthrough": true,
"min_fields": [
[
"ncc_score",
"f4"
],
[
"power",
"f4"
],
[
"template_idx",
"u4"
],
[
"angle_rad",
"f4"
]
],
"group_name": "particles",
"metafiles": [],
"description": "Passthrough from input particles.pick_stats (result_name)",
"name": "pick_stats",
"title": "Passthrough pick_stats",
"versions": [],
"type": "particle.pick_stats"
},
{
"num_items": [],
"uid": "J213-R5",
"passthrough": false,
"min_fields": [
[
"path",
"O"
],
[
"idx",
"u4"
],
[
"shape",
"2u4"
],
[
"psize_A",
"f4"
],
[
"res_A",
"f4"
]
],
"group_name": "class_averages",
"metafiles": [],
"description": "Template raw data",
"name": "blob",
"title": "Template data",
"versions": [],
"type": "template.blob"
}
],
"PID_monitor": 25862,
"cluster_job_id": null,
"interactive_hostname": "m3h015.massive.org.au",
"running_at": "2019-10-03T01:07:51.568Z",
"last_exported": "2019-10-03T00:55:07.969Z",
"ui_layouts": {
"P35": {
"show": true,
"top": 10912,
"height": 192,
"width": 444,
"groups": [],
"floater": false,
"left": 7196
},
"P35W2": {
"show": true,
"top": 10912,
"height": 192,
"width": 444,
"groups": [],
"floater": false,
"left": 6268
}
},
"no_check_inputs_ready": false,
"job_sig": "9809223528769307040381424332875339270267952113766042414826453338370109112311556718407038303802177893393480693198539638527567175353883424565377836632096554091360888008340952304156174481768639472745542961820271196906285569988785731669048404779256545944736079428169315965376632948114779001140727838097376081177623215977130554350807054573870474474420080777654600620289910931725292970669317302864731211959510423870923436933044565642007875155277826039658704562015369827036623471987805583096605789500726325701213272469231447184740088241529110153048742007667688433853361205966189345880838729138331528248341942711703254643041"
}