{
  "interpretation": "Weak-label agreement only; training fit and held-out-design protocols are separate; reviewed accuracy remains unavailable",
  "models": [
    {
      "model_id": "cnn2d_unet",
      "protocol": "leave_one_design_out",
      "datasets": 7,
      "independently_reviewed": false,
      "bce": 0.18513596083488507,
      "brier": 0.0375690328179744,
      "average_precision": 0.11050927172071054,
      "iou": 0.0,
      "dice": 0.0,
      "precision": null,
      "recall": 0.0
    },
    {
      "model_id": "cnn2d_unet",
      "protocol": "train_all",
      "datasets": 7,
      "independently_reviewed": false,
      "bce": 0.05293478032525031,
      "brier": 0.017194795732305552,
      "average_precision": 0.8868481022357211,
      "iou": 0.5131802721088435,
      "dice": 0.6471806113749848,
      "precision": 0.9047619047619048,
      "recall": 0.5148809523809523
    },
    {
      "model_id": "dgcnn",
      "protocol": "leave_one_design_out",
      "datasets": 7,
      "independently_reviewed": false,
      "bce": 0.1828704749888117,
      "brier": 0.03838243103305577,
      "average_precision": 0.06787730784318478,
      "iou": 0.0,
      "dice": 0.0,
      "precision": null,
      "recall": 0.0
    },
    {
      "model_id": "dgcnn",
      "protocol": "train_all",
      "datasets": 7,
      "independently_reviewed": false,
      "bce": 0.15873265354585303,
      "brier": 0.03609875799667397,
      "average_precision": 0.060290814641575964,
      "iou": 0.0,
      "dice": 0.0,
      "precision": null,
      "recall": 0.0
    },
    {
      "model_id": "fno2d",
      "protocol": "leave_one_design_out",
      "datasets": 7,
      "independently_reviewed": false,
      "bce": 0.31288780285530055,
      "brier": 0.0457347879441851,
      "average_precision": 0.038049094855106026,
      "iou": 0.0,
      "dice": 0.0,
      "precision": null,
      "recall": 0.0
    },
    {
      "model_id": "fno2d",
      "protocol": "train_all",
      "datasets": 7,
      "independently_reviewed": false,
      "bce": 0.06758786743493121,
      "brier": 0.02093354946451546,
      "average_precision": 0.8751340663013457,
      "iou": 0.062324929971988796,
      "dice": 0.0965121834687052,
      "precision": 0.6190476190476191,
      "recall": 0.06547619047619048
    },
    {
      "model_id": "gaussian_process",
      "protocol": "leave_one_design_out",
      "datasets": 7,
      "independently_reviewed": false,
      "bce": 0.4304539762538489,
      "brier": 0.03744950128462626,
      "average_precision": 0.08778310954899284,
      "iou": 0.0,
      "dice": 0.0,
      "precision": null,
      "recall": 0.0
    },
    {
      "model_id": "gaussian_process",
      "protocol": "train_all",
      "datasets": 7,
      "independently_reviewed": false,
      "bce": 0.15258249024584478,
      "brier": 0.026101609861962298,
      "average_precision": 0.5730350380391551,
      "iou": 0.411860396332446,
      "dice": 0.5257703081232493,
      "precision": 0.7063492063492063,
      "recall": 0.44345238095238093
    },
    {
      "model_id": "meshgraphnet",
      "protocol": "leave_one_design_out",
      "datasets": 7,
      "independently_reviewed": false,
      "bce": 0.1789993432560997,
      "brier": 0.03833411775639383,
      "average_precision": 0.0853362316380494,
      "iou": 0.0,
      "dice": 0.0,
      "precision": null,
      "recall": 0.0
    },
    {
      "model_id": "meshgraphnet",
      "protocol": "train_all",
      "datasets": 7,
      "independently_reviewed": false,
      "bce": 0.1589626779064182,
      "brier": 0.0361046866215845,
      "average_precision": 0.09777761671695151,
      "iou": 0.0,
      "dice": 0.0,
      "precision": null,
      "recall": 0.0
    },
    {
      "model_id": "physics_risk_gnn",
      "protocol": "leave_one_design_out",
      "datasets": 7,
      "independently_reviewed": false,
      "bce": 0.17917380147914228,
      "brier": 0.03840826729375517,
      "average_precision": 0.09694646120510743,
      "iou": 0.0,
      "dice": 0.0,
      "precision": null,
      "recall": 0.0
    },
    {
      "model_id": "physics_risk_gnn",
      "protocol": "train_all",
      "datasets": 7,
      "independently_reviewed": false,
      "bce": 0.15823655718342328,
      "brier": 0.03605214791064637,
      "average_precision": 0.1062333472151445,
      "iou": 0.0,
      "dice": 0.0,
      "precision": null,
      "recall": 0.0
    },
    {
      "model_id": "pointnet",
      "protocol": "leave_one_design_out",
      "datasets": 7,
      "independently_reviewed": false,
      "bce": 0.18285747326304871,
      "brier": 0.03837290092466355,
      "average_precision": 0.052835544975026294,
      "iou": 0.0,
      "dice": 0.0,
      "precision": null,
      "recall": 0.0
    },
    {
      "model_id": "pointnet",
      "protocol": "train_all",
      "datasets": 7,
      "independently_reviewed": false,
      "bce": 0.1579160453959205,
      "brier": 0.036030476093458885,
      "average_precision": 0.10404808993760477,
      "iou": 0.0,
      "dice": 0.0,
      "precision": null,
      "recall": 0.0
    },
    {
      "model_id": "tabular_mlp",
      "protocol": "leave_one_design_out",
      "datasets": 7,
      "independently_reviewed": false,
      "bce": 0.18546290583759012,
      "brier": 0.03876265234118254,
      "average_precision": 0.08097519332461665,
      "iou": 0.0,
      "dice": 0.0,
      "precision": null,
      "recall": 0.0
    },
    {
      "model_id": "tabular_mlp",
      "protocol": "train_all",
      "datasets": 7,
      "independently_reviewed": false,
      "bce": 0.15449645432561068,
      "brier": 0.03575118374655285,
      "average_precision": 0.07278562293637293,
      "iou": 0.0,
      "dice": 0.0,
      "precision": null,
      "recall": 0.0
    },
    {
      "model_id": "unet3d",
      "protocol": "leave_one_design_out",
      "datasets": 7,
      "independently_reviewed": false,
      "bce": 0.3327263846627359,
      "brier": 0.0473943058935139,
      "average_precision": 0.03314700000254661,
      "iou": 0.0,
      "dice": 0.0,
      "precision": 0.0,
      "recall": 0.0
    },
    {
      "model_id": "unet3d",
      "protocol": "train_all",
      "datasets": 7,
      "independently_reviewed": false,
      "bce": 0.031739553218533986,
      "brier": 0.010087915903472569,
      "average_precision": 0.9333357118648954,
      "iou": 0.8135254101640657,
      "dice": 0.8741496598639455,
      "precision": 0.8928571428571429,
      "recall": 0.8601190476190476
    },
    {
      "model_id": "uvnet",
      "protocol": "leave_one_design_out",
      "datasets": 7,
      "independently_reviewed": false,
      "bce": 0.20334504326065814,
      "brier": 0.0466787092555881,
      "average_precision": 0.05129173998269153,
      "iou": 0.0,
      "dice": 0.0,
      "precision": 0.0,
      "recall": 0.0
    },
    {
      "model_id": "uvnet",
      "protocol": "train_all",
      "datasets": 7,
      "independently_reviewed": false,
      "bce": 0.14782359012876906,
      "brier": 0.035376250353307805,
      "average_precision": 0.08429681757855607,
      "iou": 0.0,
      "dice": 0.0,
      "precision": null,
      "recall": 0.0
    }
  ],
  "unavailable": [
    {
      "model_id": "sparse_unet",
      "status": "unavailable_backend",
      "reason": "RuntimeError: RuntimeError: Sparse U-Net unavailable: install optional MinkowskiEngine; dense tensors are not a sparse substitute.",
      "records": 14
    },
    {
      "model_id": "sparse_unet",
      "status": "unavailable_backend",
      "reason": "RuntimeError: Sparse U-Net unavailable: install optional MinkowskiEngine; dense tensors are not a sparse substitute.",
      "records": 28
    }
  ]
}