{
  "interpretation": "Weak-label agreement only; training fit and held-out-design protocols are separate; reviewed accuracy remains unavailable",
  "models": [
    {
      "model_id": "cnn2d_unet",
      "protocol": "leave_one_design_out",
      "datasets": 6,
      "independently_reviewed": false,
      "bce": 0.43776252620118733,
      "brier": 0.04813547267850924,
      "average_precision": 0.1213026891900698,
      "iou": 0.0,
      "dice": 0.0,
      "precision": 0.0,
      "recall": 0.0
    },
    {
      "model_id": "cnn2d_unet",
      "protocol": "train_all",
      "datasets": 6,
      "independently_reviewed": false,
      "bce": 0.021065981153987345,
      "brier": 0.007409176085636585,
      "average_precision": 0.9450523356741107,
      "iou": 0.8666666666666667,
      "dice": 0.8888888888888888,
      "precision": 0.8888888888888888,
      "recall": 0.8888888888888888
    },
    {
      "model_id": "dgcnn",
      "protocol": "leave_one_design_out",
      "datasets": 6,
      "independently_reviewed": false,
      "bce": 0.19649404588119423,
      "brier": 0.04216662054061774,
      "average_precision": 0.050305775357640324,
      "iou": 0.0,
      "dice": 0.0,
      "precision": null,
      "recall": 0.0
    },
    {
      "model_id": "dgcnn",
      "protocol": "train_all",
      "datasets": 6,
      "independently_reviewed": false,
      "bce": 0.1529426981170365,
      "brier": 0.038088965326947936,
      "average_precision": 0.16542020521783612,
      "iou": 0.0,
      "dice": 0.0,
      "precision": null,
      "recall": 0.0
    },
    {
      "model_id": "fno2d",
      "protocol": "leave_one_design_out",
      "datasets": 6,
      "independently_reviewed": false,
      "bce": 0.7014898238143198,
      "brier": 0.06029528468816538,
      "average_precision": 0.043381182411962975,
      "iou": 0.0,
      "dice": 0.0,
      "precision": 0.0,
      "recall": 0.0
    },
    {
      "model_id": "fno2d",
      "protocol": "train_all",
      "datasets": 6,
      "independently_reviewed": false,
      "bce": 0.025782671112229316,
      "brier": 0.008624788372508988,
      "average_precision": 0.9411437828072245,
      "iou": 0.8621794871794872,
      "dice": 0.8933333333333334,
      "precision": 0.8846153846153847,
      "recall": 0.9027777777777778
    },
    {
      "model_id": "gaussian_process",
      "protocol": "leave_one_design_out",
      "datasets": 6,
      "independently_reviewed": false,
      "bce": 0.4529616128683609,
      "brier": 0.04123892187795603,
      "average_precision": 0.07454929443306324,
      "iou": 0.0,
      "dice": 0.0,
      "precision": null,
      "recall": 0.0
    },
    {
      "model_id": "gaussian_process",
      "protocol": "train_all",
      "datasets": 6,
      "independently_reviewed": false,
      "bce": 0.1292470902698082,
      "brier": 0.024996292238126962,
      "average_precision": 0.733238664958105,
      "iou": 0.44726800976800973,
      "dice": 0.5726495726495726,
      "precision": 0.7777777777777777,
      "recall": 0.4791666666666667
    },
    {
      "model_id": "meshgraphnet",
      "protocol": "leave_one_design_out",
      "datasets": 6,
      "independently_reviewed": false,
      "bce": 0.2085157348025668,
      "brier": 0.04244711852823,
      "average_precision": 0.06899113253347709,
      "iou": 0.0,
      "dice": 0.0,
      "precision": null,
      "recall": 0.0
    },
    {
      "model_id": "meshgraphnet",
      "protocol": "train_all",
      "datasets": 6,
      "independently_reviewed": false,
      "bce": 0.15616346636910686,
      "brier": 0.03855793512452663,
      "average_precision": 0.09156539742077575,
      "iou": 0.0,
      "dice": 0.0,
      "precision": null,
      "recall": 0.0
    },
    {
      "model_id": "physics_risk_gnn",
      "protocol": "leave_one_design_out",
      "datasets": 6,
      "independently_reviewed": false,
      "bce": 0.20596926556107553,
      "brier": 0.04275493725797874,
      "average_precision": 0.03717786437417494,
      "iou": 0.0,
      "dice": 0.0,
      "precision": null,
      "recall": 0.0
    },
    {
      "model_id": "physics_risk_gnn",
      "protocol": "train_all",
      "datasets": 6,
      "independently_reviewed": false,
      "bce": 0.15351557602944496,
      "brier": 0.03808204548966282,
      "average_precision": 0.13932222877027448,
      "iou": 0.0,
      "dice": 0.0,
      "precision": null,
      "recall": 0.0
    },
    {
      "model_id": "pointnet",
      "protocol": "leave_one_design_out",
      "datasets": 6,
      "independently_reviewed": false,
      "bce": 0.19668410917249404,
      "brier": 0.042000071079013775,
      "average_precision": 0.05170883220692444,
      "iou": 0.0,
      "dice": 0.0,
      "precision": null,
      "recall": 0.0
    },
    {
      "model_id": "pointnet",
      "protocol": "train_all",
      "datasets": 6,
      "independently_reviewed": false,
      "bce": 0.16010777132728693,
      "brier": 0.03884128330641791,
      "average_precision": 0.07781793615466254,
      "iou": 0.0,
      "dice": 0.0,
      "precision": null,
      "recall": 0.0
    },
    {
      "model_id": "tabular_mlp",
      "protocol": "leave_one_design_out",
      "datasets": 6,
      "independently_reviewed": false,
      "bce": 0.19113514829868702,
      "brier": 0.04178007333283703,
      "average_precision": 0.05007189720297287,
      "iou": 0.0,
      "dice": 0.0,
      "precision": null,
      "recall": 0.0
    },
    {
      "model_id": "tabular_mlp",
      "protocol": "train_all",
      "datasets": 6,
      "independently_reviewed": false,
      "bce": 0.157584611291352,
      "brier": 0.038589583627355345,
      "average_precision": 0.08147903994827695,
      "iou": 0.0,
      "dice": 0.0,
      "precision": null,
      "recall": 0.0
    },
    {
      "model_id": "unet3d",
      "protocol": "leave_one_design_out",
      "datasets": 6,
      "independently_reviewed": false,
      "bce": 0.46305724632659545,
      "brier": 0.04635360028952478,
      "average_precision": 0.03665533456318685,
      "iou": 0.0,
      "dice": 0.0,
      "precision": 0.0,
      "recall": 0.0
    },
    {
      "model_id": "unet3d",
      "protocol": "train_all",
      "datasets": 6,
      "independently_reviewed": false,
      "bce": 0.02764192685091801,
      "brier": 0.009684666650089215,
      "average_precision": 0.9332928346223301,
      "iou": 0.8092592592592592,
      "dice": 0.8602693602693602,
      "precision": 0.8666666666666667,
      "recall": 0.861111111111111
    },
    {
      "model_id": "uvnet",
      "protocol": "leave_one_design_out",
      "datasets": 6,
      "independently_reviewed": false,
      "bce": 0.20430207170312054,
      "brier": 0.046951898056173565,
      "average_precision": 0.22026058586983066,
      "iou": 0.08888888888888889,
      "dice": 0.13888888888888887,
      "precision": 0.4444444444444444,
      "recall": 0.1111111111111111
    },
    {
      "model_id": "uvnet",
      "protocol": "train_all",
      "datasets": 6,
      "independently_reviewed": false,
      "bce": 0.15419857230047587,
      "brier": 0.038320766376824016,
      "average_precision": 0.14762273453004812,
      "iou": 0.0,
      "dice": 0.0,
      "precision": null,
      "recall": 0.0
    }
  ],
  "unavailable": [
    {
      "model_id": "sparse_unet",
      "status": "unavailable_backend",
      "reason": "RuntimeError: RuntimeError: Sparse U-Net unavailable: install optional MinkowskiEngine; dense tensors are not a sparse substitute.",
      "records": 14
    },
    {
      "model_id": "sparse_unet",
      "status": "unavailable_backend",
      "reason": "RuntimeError: Sparse U-Net unavailable: install optional MinkowskiEngine; dense tensors are not a sparse substitute.",
      "records": 28
    }
  ]
}