| { |
| "attempt": 4, |
| "checkpoint": "${WORKSPACE}/pretraining/runs/deep44_chat_repair_04_20261109/step_0000128_5937a6def881", |
| "comparisons": [ |
| { |
| "baseline": "parent2000", |
| "confidence_interval_97_5": [ |
| 0.2868590999102137, |
| 0.3503937996078803 |
| ], |
| "gain": 0.319, |
| "gained": 331, |
| "lost": 12, |
| "method": "Scenario-cluster paired percentile bootstrap, 10000 draws; 97.5% interval for each of two baseline comparisons. Covers sampling uncertainty within this authored panel, not all chat tasks." |
| }, |
| { |
| "baseline": "attempt5", |
| "confidence_interval_97_5": [ |
| 0.22927273486383004, |
| 0.2900111982191158 |
| ], |
| "gain": 0.26, |
| "gained": 266, |
| "lost": 6, |
| "method": "Scenario-cluster paired percentile bootstrap, 10000 draws; 97.5% interval for each of two baseline comparisons. Covers sampling uncertainty within this authored panel, not all chat tasks." |
| } |
| ], |
| "legacy_test": { |
| "assistant_nll": 1.0202431740774651, |
| "components": { |
| "constraints": 0.37785995097947306, |
| "dialogue": 0.6084018315086723, |
| "general": 1.7695224359631538, |
| "grounded": 0.44701617732062005, |
| "knowledge": 1.3031791490502656, |
| "rewrite": 1.2700324356555939, |
| "skills": 0.9353672320915697, |
| "social": 1.450566180050373 |
| }, |
| "empty_rate": 0.0, |
| "generated_counts": { |
| "constraints": 128, |
| "dialogue": 128, |
| "grounded": 128, |
| "knowledge": 128, |
| "skills": 192 |
| }, |
| "generated_macro_pass": 0.5598958333333334, |
| "generated_pass_counts": { |
| "constraints": 109, |
| "dialogue": 114, |
| "grounded": 68, |
| "knowledge": 26, |
| "skills": 62 |
| }, |
| "generated_rates": { |
| "constraints": 0.8515625, |
| "dialogue": 0.890625, |
| "grounded": 0.53125, |
| "knowledge": 0.203125, |
| "skills": 0.3229166666666667 |
| }, |
| "natural_stop": 0.9865591397849462, |
| "precision": "exact-fp32-masters", |
| "qualitative_count": 40, |
| "repetition_rate": 0.020161290322580645, |
| "replacement_character_rate": 0.0, |
| "retention": { |
| "sources": { |
| "dclm": 2.9645858108997345, |
| "web": 2.8973595052957535, |
| "wiki": 2.422679301351309 |
| }, |
| "val_loss": 2.8633367460221053, |
| "validation_fingerprint": "2e9444ddc7de3cf4685911d967c1ad605d9b8cfd9244f7039621e828bf088ffa" |
| }, |
| "scope": "Checked grounded answers, deterministic tasks and authored conversation context. Public general/dialogue/social outputs require semantic review. No code, SQL, JSON, or public benchmark scores select checkpoints.", |
| "split": "test", |
| "step": 128, |
| "stop_reason_counts": { |
| "token_limit": 10, |
| "turn_end": 734 |
| }, |
| "turn_end_rate": 0.9865591397849462 |
| }, |
| "new_test": { |
| "correct": 519, |
| "counts": { |
| "abstention": 172, |
| "constraints": 25, |
| "dialogue": 194, |
| "grounded": 81, |
| "reasoning": 47 |
| }, |
| "empty": 0, |
| "false_abstention_signals": 111, |
| "identity": { |
| "data": "abcdf03853c4f00fff1e92cdf5f98d8ee3bea42526c18f15585650b368ddd2f3", |
| "evaluator": "40dbfada2856c91b0a7ffe5df7b9584175dbd479a1c6b581d1935d38dd6228aa", |
| "parent": "2118c7446c412a4ee7a4b7a356a211ad91574983e213da45980707b5fa1ca124" |
| }, |
| "missing_fact_abstention_signals": 183, |
| "natural_stop": 1.0, |
| "repetition_rate": 0.0, |
| "scoring": "Conservative whole-answer equivalence for factual tasks; exact format for transformation tasks. Strict format compliance is reported separately. Whole-passage copying and contradictory extra claims are not accepted.", |
| "strict_correct": 518, |
| "strict_counts": { |
| "abstention": 172, |
| "constraints": 25, |
| "dialogue": 194, |
| "grounded": 80, |
| "reasoning": 47 |
| }, |
| "total": 1000, |
| "totals": { |
| "abstention": 200, |
| "constraints": 200, |
| "dialogue": 200, |
| "grounded": 200, |
| "reasoning": 200 |
| } |
| }, |
| "preserved": true, |
| "public": { |
| "chat": { |
| "answerability": { |
| "False": { |
| "answer_match": 0, |
| "answer_match_with_neutral_wrapper": 0, |
| "clean_response": 15, |
| "count": 15, |
| "empty": 0, |
| "natural_stop": 15, |
| "repeated": 0, |
| "strict_task_success": 0, |
| "token_limit": 0, |
| "unexpected_structure": 0 |
| }, |
| "True": { |
| "answer_match": 12, |
| "answer_match_with_neutral_wrapper": 12, |
| "clean_response": 15, |
| "count": 15, |
| "empty": 0, |
| "natural_stop": 15, |
| "repeated": 0, |
| "strict_task_success": 12, |
| "token_limit": 0, |
| "unexpected_structure": 0 |
| } |
| }, |
| "categories": { |
| "answerability": { |
| "answer_match": 12, |
| "answer_match_with_neutral_wrapper": 12, |
| "clean_response": 30, |
| "count": 30, |
| "empty": 0, |
| "natural_stop": 30, |
| "repeated": 0, |
| "strict_task_success": 12, |
| "token_limit": 0, |
| "unexpected_structure": 0 |
| }, |
| "conversation": { |
| "answer_match": 26, |
| "answer_match_with_neutral_wrapper": 26, |
| "clean_response": 30, |
| "count": 30, |
| "empty": 0, |
| "natural_stop": 30, |
| "repeated": 0, |
| "strict_task_success": 26, |
| "token_limit": 0, |
| "unexpected_structure": 0 |
| }, |
| "instructions": { |
| "answer_match": 15, |
| "answer_match_with_neutral_wrapper": 15, |
| "clean_response": 30, |
| "count": 30, |
| "empty": 0, |
| "natural_stop": 30, |
| "repeated": 0, |
| "strict_task_success": 15, |
| "token_limit": 0, |
| "unexpected_structure": 0 |
| }, |
| "knowledge": { |
| "answer_match": 16, |
| "answer_match_with_neutral_wrapper": 16, |
| "clean_response": 30, |
| "count": 30, |
| "empty": 0, |
| "natural_stop": 30, |
| "repeated": 0, |
| "strict_task_success": 16, |
| "token_limit": 0, |
| "unexpected_structure": 0 |
| }, |
| "reading": { |
| "answer_match": 27, |
| "answer_match_with_neutral_wrapper": 27, |
| "clean_response": 30, |
| "count": 30, |
| "empty": 0, |
| "natural_stop": 30, |
| "repeated": 0, |
| "strict_task_success": 27, |
| "token_limit": 0, |
| "unexpected_structure": 0 |
| }, |
| "reasoning": { |
| "answer_match": 11, |
| "answer_match_with_neutral_wrapper": 11, |
| "clean_response": 30, |
| "count": 30, |
| "empty": 0, |
| "natural_stop": 30, |
| "repeated": 0, |
| "strict_task_success": 11, |
| "token_limit": 0, |
| "unexpected_structure": 0 |
| } |
| }, |
| "caveat": "Answer match is conservative automatic scoring, not a complete semantic correctness judgment. Non-matching answers and open chat require review. Repetition detection is heuristic.", |
| "objective": { |
| "answer_match": 107, |
| "answer_match_with_neutral_wrapper": 107, |
| "clean_response": 180, |
| "count": 180, |
| "empty": 0, |
| "natural_stop": 180, |
| "repeated": 0, |
| "strict_task_success": 107, |
| "token_limit": 0, |
| "unexpected_structure": 0 |
| }, |
| "response_control_all": { |
| "answer_match": 107, |
| "answer_match_with_neutral_wrapper": 107, |
| "clean_response": 189, |
| "count": 192, |
| "empty": 0, |
| "natural_stop": 191, |
| "repeated": 3, |
| "strict_task_success": 107, |
| "token_limit": 1, |
| "unexpected_structure": 0 |
| } |
| }, |
| "choices": { |
| "arc_challenge": { |
| "acc": 0.2935153583617747, |
| "acc_norm": 0.30887372013651876, |
| "complete": true, |
| "context_truncated_choices": 0, |
| "evaluated": 1172, |
| "expected": 1172 |
| }, |
| "arc_easy": { |
| "acc": 0.6258417508417509, |
| "acc_norm": 0.6233164983164983, |
| "complete": true, |
| "context_truncated_choices": 0, |
| "evaluated": 2376, |
| "expected": 2376 |
| }, |
| "hellaswag": { |
| "acc": 0.3285202150965943, |
| "acc_norm": 0.39404501095399325, |
| "complete": true, |
| "context_truncated_choices": 0, |
| "evaluated": 10042, |
| "expected": 10042 |
| }, |
| "piqa": { |
| "acc": 0.6648531011969532, |
| "acc_norm": 0.6621327529923831, |
| "complete": true, |
| "context_truncated_choices": 0, |
| "evaluated": 1838, |
| "expected": 1838 |
| } |
| }, |
| "complete": true, |
| "ifeval": { |
| "loose": { |
| "instruction_accuracy": 0.3249400479616307, |
| "instructions": 834, |
| "prompt_accuracy": 0.1922365988909427, |
| "prompts": 541 |
| }, |
| "strict": { |
| "instruction_accuracy": 0.3009592326139089, |
| "instructions": 834, |
| "prompt_accuracy": 0.17929759704251386, |
| "prompts": 541 |
| } |
| }, |
| "model_sha256": "7b2d8777040f05d8e0849acac4e45013b2de3d72145c363cd22825e80115c515" |
| }, |
| "reasons": [], |
| "scope": "One dev-selected finalist; reserved authored task panel is not a complete measure of conversational quality. Public and historical suites were already used in earlier experiments.", |
| "significant_on_reserved_panel_and_passed_guards": true |
| } |
|
|
