| { |
| "int8": { |
| "mb": 69.7, |
| "load_s": 0.26, |
| "ms_median": 130.5, |
| "ms_p90": 276.6, |
| "split": { |
| "n": 1142, |
| "category_acc": 0.9973730297723292, |
| "category_macro_f1": 0.9978052486323258, |
| "gate_acc": 0.9947460595446584, |
| "gate_hid": 3, |
| "gate_surfaced": 3, |
| "ask_recall": 1.0, |
| "false_finished": 0, |
| "blocked_agree": 1.0, |
| "question_line_acc": 1.0, |
| "status_line_acc": 0.9562171628721541, |
| "key_recall": 0.9566608317740086, |
| "key_precision": 0.9004602011249361, |
| "notable_agree": 0.98861646234676, |
| "busy_notable_precision": 0.9787234042553191, |
| "busy_notable_recall": 0.9517241379310345, |
| "busy_n": 412, |
| "busy_notable_n": 145, |
| "confusions": { |
| "idle->working": 2, |
| "finished->working": 1 |
| }, |
| "summary_n": 1142, |
| "summary_empty_agree": 0.936077057793345, |
| "summary_top1_in_gold": 0.782608695652174, |
| "summary_top1_exact": 0.528604118993135, |
| "summary_any_hit": 0.9107551487414187, |
| "summary_precision": 0.6945863125638406, |
| "summary_recall": 0.6914082358922217, |
| "summary_f1": 0.6929936305732484, |
| "summary_lines_per_card": 1.7145359019264448, |
| "options_exact": 0.9963503649635036, |
| "options_false_on_none": 0.005787037037037037, |
| "action_line_acc": 0.9772329246935202, |
| "task_line_acc": 0.9649737302977233, |
| "tests_acc": 0.9833625218914186, |
| "changes_acc": 0.9343257443082311, |
| "risk_acc": 0.9973730297723292, |
| "instruction_acc": 0.9929947460595446, |
| "progress_acc": 0.6471103327495622, |
| "failure_repeats_acc": 0.9273204903677759, |
| "score_acc": 0.9588441330998249, |
| "progress_within_10": 0.7994746059544658, |
| "score_within_1_anchor": 0.989492119089317 |
| }, |
| "real": { |
| "gold": { |
| "n": 60, |
| "category_coarse": 57, |
| "category_exact": 56, |
| "gate": 57, |
| "hid": 1, |
| "surfaced": 2, |
| "misses": [ |
| "c8f4b58a8e [working/claude] got approval att=0.74 want working|idle", |
| "3760524bef [working/claude] got working att=0.64 want working", |
| "7e23e9c00b [question/claude] got approval att=1.00 want question", |
| "03f3c93a3e [error/cursor] got idle att=0.07 want error" |
| ], |
| "tests": 52, |
| "changes": 51, |
| "score": 51, |
| "score_within_1_anchor": 55, |
| "task_present_agree": 24 |
| }, |
| "agents": { |
| "n": 93, |
| "category_coarse": 89, |
| "category_exact": 89, |
| "gate": 92, |
| "hid": 0, |
| "surfaced": 1, |
| "misses": [ |
| "antigravity#3 [finished/antigravity] got idle att=0.72 want finished", |
| "goose#3 [question/goose] got approval att=1.00 want question", |
| "interpreter#1 [working/interpreter] got finished att=1.00 want working", |
| "mistral#1 [question/mistral] got approval att=1.00 want question" |
| ] |
| }, |
| "summary_vs_luna": { |
| "summary_n": 153, |
| "summary_empty_agree": 0.8496732026143791, |
| "summary_top1_in_gold": 0.7870370370370371, |
| "summary_top1_exact": 0.6296296296296297, |
| "summary_any_hit": 0.8055555555555556, |
| "summary_precision": 0.7314285714285714, |
| "summary_recall": 0.6666666666666666, |
| "summary_f1": 0.6975476839237057, |
| "summary_lines_per_card": 1.1437908496732025 |
| }, |
| "lines_vs_gold": { |
| "n": 153, |
| "headline_acc": 0.8169934640522876, |
| "task_acc": 0.9869281045751634, |
| "options_exact": 0.8695652173913043, |
| "options_false_on_none": 0.009345794392523364, |
| "summary_n": 153, |
| "summary_empty_agree": 0.7843137254901961, |
| "summary_top1_in_gold": 0.6557377049180327, |
| "summary_top1_exact": 0.48360655737704916, |
| "summary_any_hit": 0.680327868852459, |
| "summary_precision": 0.6971428571428572, |
| "summary_recall": 0.5020576131687243, |
| "summary_f1": 0.583732057416268, |
| "summary_lines_per_card": 1.1437908496732025, |
| "card_any_hit": 0.8896551724137931, |
| "card_precision": 0.8221476510067114, |
| "card_recall": 0.6314432989690721, |
| "card_f1": 0.7142857142857143 |
| } |
| } |
| } |
| } |
