{
  "interpretation": "Weak-label agreement only; training fit and held-out-design protocols are separate; reviewed accuracy remains unavailable",
  "models": [
    {
      "model_id": "cnn2d_unet",
      "protocol": "leave_one_design_out",
      "datasets": 6,
      "independently_reviewed": false,
      "bce": 0.2279977556868238,
      "brier": 0.044274193355338214,
      "average_precision": 0.09786930654129256,
      "iou": 0.0,
      "dice": 0.0,
      "precision": 0.0,
      "recall": 0.0
    },
    {
      "model_id": "cnn2d_unet",
      "protocol": "train_all",
      "datasets": 6,
      "independently_reviewed": false,
      "bce": 0.07049764311660327,
      "brier": 0.023533868054269557,
      "average_precision": 0.844079297440229,
      "iou": 0.23263888888888887,
      "dice": 0.355006105006105,
      "precision": 0.8333333333333334,
      "recall": 0.23263888888888887
    },
    {
      "model_id": "dgcnn",
      "protocol": "leave_one_design_out",
      "datasets": 6,
      "independently_reviewed": false,
      "bce": 0.19250598909264793,
      "brier": 0.04178455277840734,
      "average_precision": 0.06656933312643905,
      "iou": 0.0,
      "dice": 0.0,
      "precision": null,
      "recall": 0.0
    },
    {
      "model_id": "dgcnn",
      "protocol": "train_all",
      "datasets": 6,
      "independently_reviewed": false,
      "bce": 0.1720355145685362,
      "brier": 0.03979396837787184,
      "average_precision": 0.062399580998420724,
      "iou": 0.0,
      "dice": 0.0,
      "precision": null,
      "recall": 0.0
    },
    {
      "model_id": "fno2d",
      "protocol": "leave_one_design_out",
      "datasets": 6,
      "independently_reviewed": false,
      "bce": 0.32109372503227174,
      "brier": 0.050305928535660314,
      "average_precision": 0.037709067606435574,
      "iou": 0.0,
      "dice": 0.0,
      "precision": null,
      "recall": 0.0
    },
    {
      "model_id": "fno2d",
      "protocol": "train_all",
      "datasets": 6,
      "independently_reviewed": false,
      "bce": 0.09119887685378891,
      "brier": 0.026827019944865693,
      "average_precision": 0.7725861850990946,
      "iou": 0.029411764705882356,
      "dice": 0.049999999999999996,
      "precision": 0.75,
      "recall": 0.03125
    },
    {
      "model_id": "gaussian_process",
      "protocol": "leave_one_design_out",
      "datasets": 6,
      "independently_reviewed": false,
      "bce": 0.4529616128683609,
      "brier": 0.04123892187795603,
      "average_precision": 0.07454929443306324,
      "iou": 0.0,
      "dice": 0.0,
      "precision": null,
      "recall": 0.0
    },
    {
      "model_id": "gaussian_process",
      "protocol": "train_all",
      "datasets": 6,
      "independently_reviewed": false,
      "bce": 0.1292470902698082,
      "brier": 0.024996292238126962,
      "average_precision": 0.733238664958105,
      "iou": 0.44726800976800973,
      "dice": 0.5726495726495726,
      "precision": 0.7777777777777777,
      "recall": 0.4791666666666667
    },
    {
      "model_id": "meshgraphnet",
      "protocol": "leave_one_design_out",
      "datasets": 6,
      "independently_reviewed": false,
      "bce": 0.18740107521453783,
      "brier": 0.04157829161919294,
      "average_precision": 0.0808240590441482,
      "iou": 0.0,
      "dice": 0.0,
      "precision": null,
      "recall": 0.0
    },
    {
      "model_id": "meshgraphnet",
      "protocol": "train_all",
      "datasets": 6,
      "independently_reviewed": false,
      "bce": 0.17250839340477722,
      "brier": 0.03982602456140679,
      "average_precision": 0.09209092275492137,
      "iou": 0.0,
      "dice": 0.0,
      "precision": null,
      "recall": 0.0
    },
    {
      "model_id": "physics_risk_gnn",
      "protocol": "leave_one_design_out",
      "datasets": 6,
      "independently_reviewed": false,
      "bce": 0.18793124448682397,
      "brier": 0.04167492535982883,
      "average_precision": 0.08863913889943881,
      "iou": 0.0,
      "dice": 0.0,
      "precision": null,
      "recall": 0.0
    },
    {
      "model_id": "physics_risk_gnn",
      "protocol": "train_all",
      "datasets": 6,
      "independently_reviewed": false,
      "bce": 0.17187980561142602,
      "brier": 0.03977146882135247,
      "average_precision": 0.09750250337781292,
      "iou": 0.0,
      "dice": 0.0,
      "precision": null,
      "recall": 0.0
    },
    {
      "model_id": "pointnet",
      "protocol": "leave_one_design_out",
      "datasets": 6,
      "independently_reviewed": false,
      "bce": 0.19196352470500358,
      "brier": 0.04171557974727841,
      "average_precision": 0.05753251243395063,
      "iou": 0.0,
      "dice": 0.0,
      "precision": null,
      "recall": 0.0
    },
    {
      "model_id": "pointnet",
      "protocol": "train_all",
      "datasets": 6,
      "independently_reviewed": false,
      "bce": 0.17197644959951577,
      "brier": 0.03978526260233468,
      "average_precision": 0.08861335394012453,
      "iou": 0.0,
      "dice": 0.0,
      "precision": null,
      "recall": 0.0
    },
    {
      "model_id": "tabular_mlp",
      "protocol": "leave_one_design_out",
      "datasets": 6,
      "independently_reviewed": false,
      "bce": 0.19446617990018356,
      "brier": 0.04208496860217529,
      "average_precision": 0.07841692901487192,
      "iou": 0.0,
      "dice": 0.0,
      "precision": null,
      "recall": 0.0
    },
    {
      "model_id": "tabular_mlp",
      "protocol": "train_all",
      "datasets": 6,
      "independently_reviewed": false,
      "bce": 0.1691775308196611,
      "brier": 0.03954215937729302,
      "average_precision": 0.07105340834462419,
      "iou": 0.0,
      "dice": 0.0,
      "precision": null,
      "recall": 0.0
    },
    {
      "model_id": "unet3d",
      "protocol": "leave_one_design_out",
      "datasets": 6,
      "independently_reviewed": false,
      "bce": 0.3506855297256541,
      "brier": 0.04981724921878266,
      "average_precision": 0.034806628413471166,
      "iou": 0.0,
      "dice": 0.0,
      "precision": null,
      "recall": 0.0
    },
    {
      "model_id": "unet3d",
      "protocol": "train_all",
      "datasets": 6,
      "independently_reviewed": false,
      "bce": 0.04267640609899201,
      "brier": 0.013349599656770022,
      "average_precision": 0.9092644383501827,
      "iou": 0.756452153510977,
      "dice": 0.8262924814648954,
      "precision": 0.8650997150997152,
      "recall": 0.7951388888888888
    },
    {
      "model_id": "uvnet",
      "protocol": "leave_one_design_out",
      "datasets": 6,
      "independently_reviewed": false,
      "bce": 0.2128878965871843,
      "brier": 0.049287130617760605,
      "average_precision": 0.05622642906295446,
      "iou": 0.0,
      "dice": 0.0,
      "precision": 0.0,
      "recall": 0.0
    },
    {
      "model_id": "uvnet",
      "protocol": "train_all",
      "datasets": 6,
      "independently_reviewed": false,
      "bce": 0.16909284146781633,
      "brier": 0.040032581938249545,
      "average_precision": 0.09495512203184503,
      "iou": 0.0,
      "dice": 0.0,
      "precision": null,
      "recall": 0.0
    }
  ],
  "unavailable": [
    {
      "model_id": "sparse_unet",
      "status": "unavailable_backend",
      "reason": "RuntimeError: RuntimeError: Sparse U-Net unavailable: install optional MinkowskiEngine; dense tensors are not a sparse substitute.",
      "records": 14
    },
    {
      "model_id": "sparse_unet",
      "status": "unavailable_backend",
      "reason": "RuntimeError: Sparse U-Net unavailable: install optional MinkowskiEngine; dense tensors are not a sparse substitute.",
      "records": 28
    }
  ]
}