_summary.json

{
  "failures": [],
  "num_completed": 1,
  "num_failed": 0,
  "results": {
    "SH": {
      "_ablation_id": "SH",
      "_deltas": {
        "merge_heal_enabled": true,
        "merge_heal_train_router": true
      },
      "_elapsed_seconds": 1607.625530552,
      "acc_norm_sum_gap": -0.06500000000000017,
      "bpt_gap": 0.5943762334589815,
      "corpus": {
        "dataset": "wikitext",
        "name": "wikitext",
        "num_sequences": 64,
        "sequence_length": 2048,
        "split": "test",
        "subset": "wikitext-2-raw-v1"
      },
      "lm_eval": {
        "arc_easy_limit": 100,
        "hellaswag_limit": 200
      },
      "mode": "thermometer",
      "stage": "6alt",
      "student_acc_norm_sum": 1.4249999999999998,
      "student_arc_easy_acc_norm": 0.73,
      "student_bpt": 3.318567814998621,
      "student_hellaswag_acc_norm": 0.695,
      "teacher_acc_norm_sum": 1.49,
      "teacher_arc_easy_acc_norm": 0.75,
      "teacher_bpt": 2.7241915815396394,
      "teacher_cache": {
        "hit": true,
        "key": "1de59f44ecc5909fe27f32a459674bc96f4fa318564914a33e5c3b9a5f500830",
        "path": "/cache/ablations/_shared/thermometer_teacher_cache.json"
      },
      "teacher_hellaswag_acc_norm": 0.74,
      "top1_agreement": 0.7315889000892639
    }
  },
  "stage6_mode": "thermometer",
  "total_ablations_planned": 1
}