{
  "interpretation": "Weak-label agreement only; training fit and held-out-design protocols are separate; reviewed accuracy remains unavailable",
  "models": [
    {
      "model_id": "cnn2d_unet",
      "protocol": "per_design",
      "datasets": 1,
      "independently_reviewed": false,
      "bce": 0.07862466750951398,
      "brier": 0.014364444244999011,
      "average_precision": 0.07914242620124973,
      "iou": 0.0,
      "dice": 0.0,
      "precision": null,
      "recall": 0.0
    },
    {
      "model_id": "cnn2d_unet",
      "protocol": "train_all",
      "datasets": 1,
      "independently_reviewed": false,
      "bce": 0.07862466750951398,
      "brier": 0.014364444244999011,
      "average_precision": 0.07914242620124973,
      "iou": 0.0,
      "dice": 0.0,
      "precision": null,
      "recall": 0.0
    },
    {
      "model_id": "dgcnn",
      "protocol": "per_design",
      "datasets": 1,
      "independently_reviewed": false,
      "bce": 0.6795382407413781,
      "brier": 0.24319597861925046,
      "average_precision": 0.010165385801371002,
      "iou": 0.0,
      "dice": 0.0,
      "precision": null,
      "recall": 0.0
    },
    {
      "model_id": "dgcnn",
      "protocol": "train_all",
      "datasets": 1,
      "independently_reviewed": false,
      "bce": 0.6795382407413781,
      "brier": 0.24319597861925046,
      "average_precision": 0.010165385801371002,
      "iou": 0.0,
      "dice": 0.0,
      "precision": null,
      "recall": 0.0
    },
    {
      "model_id": "fno2d",
      "protocol": "per_design",
      "datasets": 1,
      "independently_reviewed": false,
      "bce": 0.5131083899415718,
      "brier": 0.16117509529587798,
      "average_precision": 0.02564303978503944,
      "iou": 0.0,
      "dice": 0.0,
      "precision": null,
      "recall": 0.0
    },
    {
      "model_id": "fno2d",
      "protocol": "train_all",
      "datasets": 1,
      "independently_reviewed": false,
      "bce": 0.5131083899415718,
      "brier": 0.16117509529587798,
      "average_precision": 0.02564303978503944,
      "iou": 0.0,
      "dice": 0.0,
      "precision": null,
      "recall": 0.0
    },
    {
      "model_id": "gaussian_process",
      "protocol": "per_design",
      "datasets": 1,
      "independently_reviewed": false,
      "bce": 0.009305351252086747,
      "brier": 0.0012484752882059012,
      "average_precision": 1.0,
      "iou": 1.0,
      "dice": 1.0,
      "precision": 1.0,
      "recall": 1.0
    },
    {
      "model_id": "gaussian_process",
      "protocol": "train_all",
      "datasets": 1,
      "independently_reviewed": false,
      "bce": 0.009305351252086747,
      "brier": 0.0012484752882059012,
      "average_precision": 1.0,
      "iou": 1.0,
      "dice": 1.0,
      "precision": 1.0,
      "recall": 1.0
    },
    {
      "model_id": "meshgraphnet",
      "protocol": "per_design",
      "datasets": 1,
      "independently_reviewed": false,
      "bce": 0.4799083885409725,
      "brier": 0.1454935320951523,
      "average_precision": 0.08062982768865122,
      "iou": 0.0,
      "dice": 0.0,
      "precision": null,
      "recall": 0.0
    },
    {
      "model_id": "meshgraphnet",
      "protocol": "train_all",
      "datasets": 1,
      "independently_reviewed": false,
      "bce": 0.4799083885409725,
      "brier": 0.1454935320951523,
      "average_precision": 0.08062982768865122,
      "iou": 0.0,
      "dice": 0.0,
      "precision": null,
      "recall": 0.0
    },
    {
      "model_id": "physics_risk_gnn",
      "protocol": "per_design",
      "datasets": 1,
      "independently_reviewed": false,
      "bce": 0.413608732142603,
      "brier": 0.11515660878213368,
      "average_precision": 0.07513061650992686,
      "iou": 0.0,
      "dice": 0.0,
      "precision": null,
      "recall": 0.0
    },
    {
      "model_id": "physics_risk_gnn",
      "protocol": "train_all",
      "datasets": 1,
      "independently_reviewed": false,
      "bce": 0.413608732142603,
      "brier": 0.11515660878213368,
      "average_precision": 0.07513061650992686,
      "iou": 0.0,
      "dice": 0.0,
      "precision": null,
      "recall": 0.0
    },
    {
      "model_id": "pointnet",
      "protocol": "per_design",
      "datasets": 1,
      "independently_reviewed": false,
      "bce": 0.4606565603936074,
      "brier": 0.13642984078986525,
      "average_precision": 0.031836777197601945,
      "iou": 0.0,
      "dice": 0.0,
      "precision": null,
      "recall": 0.0
    },
    {
      "model_id": "pointnet",
      "protocol": "train_all",
      "datasets": 1,
      "independently_reviewed": false,
      "bce": 0.4606565603936074,
      "brier": 0.13642984078986525,
      "average_precision": 0.031836777197601945,
      "iou": 0.0,
      "dice": 0.0,
      "precision": null,
      "recall": 0.0
    },
    {
      "model_id": "tabular_mlp",
      "protocol": "per_design",
      "datasets": 1,
      "independently_reviewed": false,
      "bce": 0.25984676425495973,
      "brier": 0.05401715678434716,
      "average_precision": 0.020530382764842342,
      "iou": 0.0,
      "dice": 0.0,
      "precision": null,
      "recall": 0.0
    },
    {
      "model_id": "tabular_mlp",
      "protocol": "train_all",
      "datasets": 1,
      "independently_reviewed": false,
      "bce": 0.25984676425495973,
      "brier": 0.05401715678434716,
      "average_precision": 0.020530382764842342,
      "iou": 0.0,
      "dice": 0.0,
      "precision": null,
      "recall": 0.0
    },
    {
      "model_id": "unet3d",
      "protocol": "per_design",
      "datasets": 1,
      "independently_reviewed": false,
      "bce": 0.41843373308654713,
      "brier": 0.11870174886604659,
      "average_precision": 0.031578947368421054,
      "iou": 0.0,
      "dice": 0.0,
      "precision": null,
      "recall": 0.0
    },
    {
      "model_id": "unet3d",
      "protocol": "train_all",
      "datasets": 1,
      "independently_reviewed": false,
      "bce": 0.41843373308654713,
      "brier": 0.11870174886604659,
      "average_precision": 0.031578947368421054,
      "iou": 0.0,
      "dice": 0.0,
      "precision": null,
      "recall": 0.0
    },
    {
      "model_id": "uvnet",
      "protocol": "per_design",
      "datasets": 1,
      "independently_reviewed": false,
      "bce": 0.10360554164989992,
      "brier": 0.01336088283801694,
      "average_precision": 0.036325576173773515,
      "iou": 0.0,
      "dice": 0.0,
      "precision": null,
      "recall": 0.0
    },
    {
      "model_id": "uvnet",
      "protocol": "train_all",
      "datasets": 1,
      "independently_reviewed": false,
      "bce": 0.10360554164989992,
      "brier": 0.01336088283801694,
      "average_precision": 0.036325576173773515,
      "iou": 0.0,
      "dice": 0.0,
      "precision": null,
      "recall": 0.0
    }
  ],
  "unavailable": [
    {
      "model_id": "sparse_unet",
      "status": "unavailable_backend",
      "reason": "RuntimeError: RuntimeError: Sparse U-Net unavailable: install optional MinkowskiEngine; dense tensors are not a sparse substitute.",
      "records": 30
    },
    {
      "model_id": "sparse_unet",
      "status": "unavailable_backend",
      "reason": "RuntimeError: Sparse U-Net unavailable: install optional MinkowskiEngine; dense tensors are not a sparse substitute.",
      "records": 90
    }
  ]
}