{
  "status": "saved_same_run_weak_label_risk_comparison",
  "production_valid": false,
  "part_run_count": {
    "AC28": 2,
    "ME1a_DT": 1
  },
  "evaluation": "CAD-mapped 10 mm CSV-seeded weak risk mask on AC28 18h01 and ME1a 14h06 training runs",
  "held_out_baselines_available": false,
  "limitations": [
    "Unvalidated CAD-to-scan registration and defect extents",
    "Different training epochs, source domains and CAD mapping support",
    "Risk GNN was trained on two AC28 runs; other prior AC28 models used 18h01",
    "Optimized residual GNN predicts risk only; Risk GNN also predicts deformation"
  ],
  "rows": [
    {
      "model": "2D CNN/U-Net",
      "family": "prior baseline",
      "graph_nodes_requested": "",
      "ac28_ap": 0.6224617546673825,
      "ac28_iou": 0.3333333333333333,
      "ac28_dice": 0.5,
      "ac28_coverage": 0.9872030779524925,
      "ac28_epochs": 80,
      "ac28_training_runs": "AC28 18h01",
      "me1a_ap": 0.2978150926426788,
      "me1a_iou": 0.14285714285714285,
      "me1a_dice": 0.25,
      "me1a_coverage": 0.9216975008114249,
      "me1a_epochs": 80,
      "me1a_training_runs": "ME1a 14h06",
      "ac28_checkpoint": "results/models/cad_predictions/checkpoints/18h01-2d-cnn-u-net.pt",
      "me1a_checkpoint": "experiments/ME1a_DT/results/models/cad_predictions/checkpoints/14h06-2d-cnn-u-net.pt"
    },
    {
      "model": "FNO2d",
      "family": "prior baseline",
      "graph_nodes_requested": "",
      "ac28_ap": 0.6197336024268073,
      "ac28_iou": 0.2962962962962963,
      "ac28_dice": 0.45714285714285713,
      "ac28_coverage": 0.9872030779524925,
      "ac28_epochs": 60,
      "ac28_training_runs": "AC28 18h01",
      "me1a_ap": 0.44375168386007535,
      "me1a_iou": 0.14285714285714285,
      "me1a_dice": 0.25,
      "me1a_coverage": 0.9216975008114249,
      "me1a_epochs": 60,
      "me1a_training_runs": "ME1a 14h06",
      "ac28_checkpoint": "results/models/cad_predictions/checkpoints/18h01-fno2d.pt",
      "me1a_checkpoint": "experiments/ME1a_DT/results/models/cad_predictions/checkpoints/14h06-fno2d.pt"
    },
    {
      "model": "Dense 3D U-Net",
      "family": "prior baseline",
      "graph_nodes_requested": "",
      "ac28_ap": 0.3584950358630279,
      "ac28_iou": 0.04849884526558892,
      "ac28_dice": 0.09251101321585903,
      "ac28_coverage": 1.0,
      "ac28_epochs": 100,
      "ac28_training_runs": "AC28 18h01",
      "me1a_ap": 0.09726685404303197,
      "me1a_iou": 0.018950437317784258,
      "me1a_dice": 0.03719599427753934,
      "me1a_coverage": 1.0,
      "me1a_epochs": 100,
      "me1a_training_runs": "ME1a 14h06",
      "ac28_checkpoint": "results/models/cad_predictions/checkpoints/18h01-dense-3d-u-net.pt",
      "me1a_checkpoint": "experiments/ME1a_DT/results/models/cad_predictions/checkpoints/14h06-dense-3d-u-net.pt"
    },
    {
      "model": "Sparse 3D U-Net",
      "family": "prior baseline",
      "graph_nodes_requested": "",
      "ac28_ap": 0.35856212384220376,
      "ac28_iou": 0.04740406320541761,
      "ac28_dice": 0.09051724137931035,
      "ac28_coverage": 1.0,
      "ac28_epochs": 100,
      "ac28_training_runs": "AC28 18h01",
      "me1a_ap": 0.11249807332160783,
      "me1a_iou": 0.01867816091954023,
      "me1a_dice": 0.036671368124118475,
      "me1a_coverage": 1.0,
      "me1a_epochs": 100,
      "me1a_training_runs": "ME1a 14h06",
      "ac28_checkpoint": "results/models/cad_predictions/checkpoints/18h01-sparse-3d-u-net.pt",
      "me1a_checkpoint": "experiments/ME1a_DT/results/models/cad_predictions/checkpoints/14h06-sparse-3d-u-net.pt"
    },
    {
      "model": "PointNet",
      "family": "prior baseline",
      "graph_nodes_requested": "",
      "ac28_ap": 0.09272106993537317,
      "ac28_iou": 0.11363636363636363,
      "ac28_dice": 0.20408163265306123,
      "ac28_coverage": 0.9999163599866177,
      "ac28_epochs": 120,
      "ac28_training_runs": "AC28 18h01",
      "me1a_ap": 0.07607437273496566,
      "me1a_iou": 0.008620689655172414,
      "me1a_dice": 0.017094017094017096,
      "me1a_coverage": 0.9974034404414152,
      "me1a_epochs": 120,
      "me1a_training_runs": "ME1a 14h06",
      "ac28_checkpoint": "results/models/cad_predictions/checkpoints/18h01-pointnet.pt",
      "me1a_checkpoint": "experiments/ME1a_DT/results/models/cad_predictions/checkpoints/14h06-pointnet.pt"
    },
    {
      "model": "DGCNN",
      "family": "prior baseline",
      "graph_nodes_requested": "",
      "ac28_ap": 0.002008270225324489,
      "ac28_iou": 0.0,
      "ac28_dice": 0.0,
      "ac28_coverage": 0.7357811977249916,
      "ac28_epochs": 100,
      "ac28_training_runs": "AC28 18h01",
      "me1a_ap": 0.038238033630332416,
      "me1a_iou": 0.06521739130434782,
      "me1a_dice": 0.12244897959183673,
      "me1a_coverage": 0.6301525478740668,
      "me1a_epochs": 100,
      "me1a_training_runs": "ME1a 14h06",
      "ac28_checkpoint": "results/models/cad_predictions/checkpoints/18h01-dgcnn.pt",
      "me1a_checkpoint": "experiments/ME1a_DT/results/models/cad_predictions/checkpoints/14h06-dgcnn.pt"
    },
    {
      "model": "Mesh GNN",
      "family": "prior baseline",
      "graph_nodes_requested": "",
      "ac28_ap": 0.7575047512332862,
      "ac28_iou": 0.14685314685314685,
      "ac28_dice": 0.25609756097560976,
      "ac28_coverage": 1.0,
      "ac28_epochs": 100,
      "ac28_training_runs": "AC28 18h01",
      "me1a_ap": 0.04479293965435879,
      "me1a_iou": 0.020618556701030927,
      "me1a_dice": 0.04040404040404041,
      "me1a_coverage": 1.0,
      "me1a_epochs": 100,
      "me1a_training_runs": "ME1a 14h06",
      "ac28_checkpoint": "results/models/cad_predictions/checkpoints/18h01-mesh-gnn.pt",
      "me1a_checkpoint": "experiments/ME1a_DT/results/models/cad_predictions/checkpoints/14h06-mesh-gnn.pt"
    },
    {
      "model": "Risk GNN",
      "family": "prior baseline",
      "graph_nodes_requested": "",
      "ac28_ap": 0.7642630053466928,
      "ac28_iou": 0.3088235294117647,
      "ac28_dice": 0.47191011235955055,
      "ac28_coverage": 1.0,
      "ac28_epochs": 400,
      "ac28_training_runs": "AC28 16h + 18h01",
      "me1a_ap": 0.2529800405464673,
      "me1a_iou": 0.036011080332409975,
      "me1a_dice": 0.06951871657754011,
      "me1a_coverage": 1.0,
      "me1a_epochs": 400,
      "me1a_training_runs": "ME1a 14h06",
      "ac28_checkpoint": "results/models/cad_predictions/checkpoints/18h01-risk-gnn.pt",
      "me1a_checkpoint": "experiments/ME1a_DT/results/models/cad_predictions/checkpoints/14h06-risk-gnn.pt"
    },
    {
      "model": "UV-Net",
      "family": "prior baseline",
      "graph_nodes_requested": "",
      "ac28_ap": 0.07645003258129793,
      "ac28_iou": 0.04259634888438134,
      "ac28_dice": 0.08171206225680934,
      "ac28_coverage": 0.9790063566410171,
      "ac28_epochs": 100,
      "ac28_training_runs": "AC28 18h01",
      "me1a_ap": 0.01149431921692329,
      "me1a_iou": 0.010526315789473684,
      "me1a_dice": 0.020833333333333332,
      "me1a_coverage": 0.9658390133073678,
      "me1a_epochs": 100,
      "me1a_training_runs": "ME1a 14h06",
      "ac28_checkpoint": "results/models/cad_predictions/checkpoints/18h01-uv-net.pt",
      "me1a_checkpoint": "experiments/ME1a_DT/results/models/cad_predictions/checkpoints/14h06-uv-net.pt"
    },
    {
      "model": "Tabular MLP",
      "family": "prior baseline",
      "graph_nodes_requested": "",
      "ac28_ap": 0.001756440281030445,
      "ac28_iou": 0.0,
      "ac28_dice": 0.0,
      "ac28_coverage": 1.0,
      "ac28_epochs": 120,
      "ac28_training_runs": "AC28 18h01",
      "me1a_ap": 0.0010548523206751054,
      "me1a_iou": 0.0,
      "me1a_dice": 0.0,
      "me1a_coverage": 1.0,
      "me1a_epochs": 120,
      "me1a_training_runs": "ME1a 14h06",
      "ac28_checkpoint": "results/models/cad_predictions/checkpoints/18h01-tabular-mlp.pt",
      "me1a_checkpoint": "experiments/ME1a_DT/results/models/cad_predictions/checkpoints/14h06-tabular-mlp.pt"
    },
    {
      "model": "Residual Geo GNN (individual)",
      "family": "optimized risk-only GNN",
      "graph_nodes_requested": 12000,
      "ac28_ap": 0.44554569627256424,
      "ac28_iou": 0.13636363636363635,
      "ac28_dice": 0.24,
      "ac28_coverage": 1.0,
      "ac28_epochs": 180,
      "ac28_training_runs": "AC28_18h01",
      "me1a_ap": 0.6292425720881771,
      "me1a_iou": 0.47619047619047616,
      "me1a_dice": 0.6451612903225806,
      "me1a_coverage": 1.0,
      "me1a_epochs": 180,
      "me1a_training_runs": "ME1a_14h06",
      "ac28_checkpoint": "results/ME1a_DT_comparison/gnn_joint_hard/residual_geo_12000_single_AC28_18h01.pt",
      "me1a_checkpoint": "results/ME1a_DT_comparison/gnn_joint_hard/residual_geo_12000_single_ME1a_14h06.pt"
    },
    {
      "model": "Residual Geo GNN (joint)",
      "family": "optimized risk-only GNN",
      "graph_nodes_requested": 12000,
      "ac28_ap": 0.6714374567054266,
      "ac28_iou": 0.46511627906976744,
      "ac28_dice": 0.6349206349206349,
      "ac28_coverage": 1.0,
      "ac28_epochs": 180,
      "ac28_training_runs": "AC28_16h+AC28_18h01+ME1a_14h06",
      "me1a_ap": 0.476497972396576,
      "me1a_iou": 0.053497942386831275,
      "me1a_dice": 0.1015625,
      "me1a_coverage": 1.0,
      "me1a_epochs": 180,
      "me1a_training_runs": "AC28_16h+AC28_18h01+ME1a_14h06",
      "ac28_checkpoint": "results/ME1a_DT_comparison/gnn_joint_hard/residual_geo_12000_joint.pt",
      "me1a_checkpoint": "results/ME1a_DT_comparison/gnn_joint_hard/residual_geo_12000_joint.pt"
    }
  ]
}