{
  "attempt": 4,
  "checkpoint": "${WORKSPACE}/pretraining/runs/deep44_chat_repair_04_20261109/step_0000128_5937a6def881",
  "comparisons": [
    {
      "baseline": "parent2000",
      "confidence_interval_97_5": [
        0.2868590999102137,
        0.3503937996078803
      ],
      "gain": 0.319,
      "gained": 331,
      "lost": 12,
      "method": "Scenario-cluster paired percentile bootstrap, 10000 draws; 97.5% interval for each of two baseline comparisons. Covers sampling uncertainty within this authored panel, not all chat tasks."
    },
    {
      "baseline": "attempt5",
      "confidence_interval_97_5": [
        0.22927273486383004,
        0.2900111982191158
      ],
      "gain": 0.26,
      "gained": 266,
      "lost": 6,
      "method": "Scenario-cluster paired percentile bootstrap, 10000 draws; 97.5% interval for each of two baseline comparisons. Covers sampling uncertainty within this authored panel, not all chat tasks."
    }
  ],
  "legacy_test": {
    "assistant_nll": 1.0202431740774651,
    "components": {
      "constraints": 0.37785995097947306,
      "dialogue": 0.6084018315086723,
      "general": 1.7695224359631538,
      "grounded": 0.44701617732062005,
      "knowledge": 1.3031791490502656,
      "rewrite": 1.2700324356555939,
      "skills": 0.9353672320915697,
      "social": 1.450566180050373
    },
    "empty_rate": 0.0,
    "generated_counts": {
      "constraints": 128,
      "dialogue": 128,
      "grounded": 128,
      "knowledge": 128,
      "skills": 192
    },
    "generated_macro_pass": 0.5598958333333334,
    "generated_pass_counts": {
      "constraints": 109,
      "dialogue": 114,
      "grounded": 68,
      "knowledge": 26,
      "skills": 62
    },
    "generated_rates": {
      "constraints": 0.8515625,
      "dialogue": 0.890625,
      "grounded": 0.53125,
      "knowledge": 0.203125,
      "skills": 0.3229166666666667
    },
    "natural_stop": 0.9865591397849462,
    "precision": "exact-fp32-masters",
    "qualitative_count": 40,
    "repetition_rate": 0.020161290322580645,
    "replacement_character_rate": 0.0,
    "retention": {
      "sources": {
        "dclm": 2.9645858108997345,
        "web": 2.8973595052957535,
        "wiki": 2.422679301351309
      },
      "val_loss": 2.8633367460221053,
      "validation_fingerprint": "2e9444ddc7de3cf4685911d967c1ad605d9b8cfd9244f7039621e828bf088ffa"
    },
    "scope": "Checked grounded answers, deterministic tasks and authored conversation context. Public general/dialogue/social outputs require semantic review. No code, SQL, JSON, or public benchmark scores select checkpoints.",
    "split": "test",
    "step": 128,
    "stop_reason_counts": {
      "token_limit": 10,
      "turn_end": 734
    },
    "turn_end_rate": 0.9865591397849462
  },
  "new_test": {
    "correct": 519,
    "counts": {
      "abstention": 172,
      "constraints": 25,
      "dialogue": 194,
      "grounded": 81,
      "reasoning": 47
    },
    "empty": 0,
    "false_abstention_signals": 111,
    "identity": {
      "data": "abcdf03853c4f00fff1e92cdf5f98d8ee3bea42526c18f15585650b368ddd2f3",
      "evaluator": "40dbfada2856c91b0a7ffe5df7b9584175dbd479a1c6b581d1935d38dd6228aa",
      "parent": "2118c7446c412a4ee7a4b7a356a211ad91574983e213da45980707b5fa1ca124"
    },
    "missing_fact_abstention_signals": 183,
    "natural_stop": 1.0,
    "repetition_rate": 0.0,
    "scoring": "Conservative whole-answer equivalence for factual tasks; exact format for transformation tasks. Strict format compliance is reported separately. Whole-passage copying and contradictory extra claims are not accepted.",
    "strict_correct": 518,
    "strict_counts": {
      "abstention": 172,
      "constraints": 25,
      "dialogue": 194,
      "grounded": 80,
      "reasoning": 47
    },
    "total": 1000,
    "totals": {
      "abstention": 200,
      "constraints": 200,
      "dialogue": 200,
      "grounded": 200,
      "reasoning": 200
    }
  },
  "preserved": true,
  "public": {
    "chat": {
      "answerability": {
        "False": {
          "answer_match": 0,
          "answer_match_with_neutral_wrapper": 0,
          "clean_response": 15,
          "count": 15,
          "empty": 0,
          "natural_stop": 15,
          "repeated": 0,
          "strict_task_success": 0,
          "token_limit": 0,
          "unexpected_structure": 0
        },
        "True": {
          "answer_match": 12,
          "answer_match_with_neutral_wrapper": 12,
          "clean_response": 15,
          "count": 15,
          "empty": 0,
          "natural_stop": 15,
          "repeated": 0,
          "strict_task_success": 12,
          "token_limit": 0,
          "unexpected_structure": 0
        }
      },
      "categories": {
        "answerability": {
          "answer_match": 12,
          "answer_match_with_neutral_wrapper": 12,
          "clean_response": 30,
          "count": 30,
          "empty": 0,
          "natural_stop": 30,
          "repeated": 0,
          "strict_task_success": 12,
          "token_limit": 0,
          "unexpected_structure": 0
        },
        "conversation": {
          "answer_match": 26,
          "answer_match_with_neutral_wrapper": 26,
          "clean_response": 30,
          "count": 30,
          "empty": 0,
          "natural_stop": 30,
          "repeated": 0,
          "strict_task_success": 26,
          "token_limit": 0,
          "unexpected_structure": 0
        },
        "instructions": {
          "answer_match": 15,
          "answer_match_with_neutral_wrapper": 15,
          "clean_response": 30,
          "count": 30,
          "empty": 0,
          "natural_stop": 30,
          "repeated": 0,
          "strict_task_success": 15,
          "token_limit": 0,
          "unexpected_structure": 0
        },
        "knowledge": {
          "answer_match": 16,
          "answer_match_with_neutral_wrapper": 16,
          "clean_response": 30,
          "count": 30,
          "empty": 0,
          "natural_stop": 30,
          "repeated": 0,
          "strict_task_success": 16,
          "token_limit": 0,
          "unexpected_structure": 0
        },
        "reading": {
          "answer_match": 27,
          "answer_match_with_neutral_wrapper": 27,
          "clean_response": 30,
          "count": 30,
          "empty": 0,
          "natural_stop": 30,
          "repeated": 0,
          "strict_task_success": 27,
          "token_limit": 0,
          "unexpected_structure": 0
        },
        "reasoning": {
          "answer_match": 11,
          "answer_match_with_neutral_wrapper": 11,
          "clean_response": 30,
          "count": 30,
          "empty": 0,
          "natural_stop": 30,
          "repeated": 0,
          "strict_task_success": 11,
          "token_limit": 0,
          "unexpected_structure": 0
        }
      },
      "caveat": "Answer match is conservative automatic scoring, not a complete semantic correctness judgment. Non-matching answers and open chat require review. Repetition detection is heuristic.",
      "objective": {
        "answer_match": 107,
        "answer_match_with_neutral_wrapper": 107,
        "clean_response": 180,
        "count": 180,
        "empty": 0,
        "natural_stop": 180,
        "repeated": 0,
        "strict_task_success": 107,
        "token_limit": 0,
        "unexpected_structure": 0
      },
      "response_control_all": {
        "answer_match": 107,
        "answer_match_with_neutral_wrapper": 107,
        "clean_response": 189,
        "count": 192,
        "empty": 0,
        "natural_stop": 191,
        "repeated": 3,
        "strict_task_success": 107,
        "token_limit": 1,
        "unexpected_structure": 0
      }
    },
    "choices": {
      "arc_challenge": {
        "acc": 0.2935153583617747,
        "acc_norm": 0.30887372013651876,
        "complete": true,
        "context_truncated_choices": 0,
        "evaluated": 1172,
        "expected": 1172
      },
      "arc_easy": {
        "acc": 0.6258417508417509,
        "acc_norm": 0.6233164983164983,
        "complete": true,
        "context_truncated_choices": 0,
        "evaluated": 2376,
        "expected": 2376
      },
      "hellaswag": {
        "acc": 0.3285202150965943,
        "acc_norm": 0.39404501095399325,
        "complete": true,
        "context_truncated_choices": 0,
        "evaluated": 10042,
        "expected": 10042
      },
      "piqa": {
        "acc": 0.6648531011969532,
        "acc_norm": 0.6621327529923831,
        "complete": true,
        "context_truncated_choices": 0,
        "evaluated": 1838,
        "expected": 1838
      }
    },
    "complete": true,
    "ifeval": {
      "loose": {
        "instruction_accuracy": 0.3249400479616307,
        "instructions": 834,
        "prompt_accuracy": 0.1922365988909427,
        "prompts": 541
      },
      "strict": {
        "instruction_accuracy": 0.3009592326139089,
        "instructions": 834,
        "prompt_accuracy": 0.17929759704251386,
        "prompts": 541
      }
    },
    "model_sha256": "7b2d8777040f05d8e0849acac4e45013b2de3d72145c363cd22825e80115c515"
  },
  "reasons": [],
  "scope": "One dev-selected finalist; reserved authored task panel is not a complete measure of conversational quality. Public and historical suites were already used in earlier experiments.",
  "significant_on_reserved_panel_and_passed_guards": true
}