| { |
| "int8": { |
| "mb": 17.4, |
| "load_s": 0.07, |
| "ms_median": 21.4, |
| "ms_p90": 44.6, |
| "split": { |
| "n": 1142, |
| "category_acc": 0.9956217162872154, |
| "category_macro_f1": 0.9951614882286082, |
| "gate_acc": 0.9947460595446584, |
| "gate_hid": 2, |
| "gate_surfaced": 4, |
| "ask_recall": 0.9966666666666667, |
| "false_finished": 1, |
| "blocked_agree": 0.999124343257443, |
| "question_line_acc": 1.0, |
| "status_line_acc": 0.7880910683012259, |
| "key_recall": 0.957868050944649, |
| "key_precision": 0.8764497956478515, |
| "notable_agree": 0.9772329246935202, |
| "busy_notable_precision": 0.9416058394160584, |
| "busy_notable_recall": 0.8896551724137931, |
| "busy_n": 412, |
| "busy_notable_n": 145, |
| "confusions": { |
| "approval->finished": 1, |
| "error->working": 1, |
| "finished->error": 1, |
| "finished->working": 1, |
| "idle->working": 1 |
| }, |
| "card_first_good": 0.9098073555166375, |
| "card_first_best": 0.797723292469352, |
| "card_noise_rate": 0.13994743758212877, |
| "card_lines_per_card": 1.3327495621716288, |
| "card_empty_when_something": 0.0, |
| "summary_n": 1142, |
| "summary_empty_agree": 0.9115586690017513, |
| "summary_top1_in_gold": 0.2631578947368421, |
| "summary_top1_exact": 0.15789473684210525, |
| "summary_any_hit": 0.6681922196796338, |
| "summary_precision": 0.40519877675840976, |
| "summary_recall": 0.4041687849517031, |
| "summary_f1": 0.4046831254772207, |
| "summary_lines_per_card": 1.7180385288966724, |
| "options_exact": 0.9452554744525548, |
| "options_false_on_none": 0.004629629629629629, |
| "action_line_acc": 0.978108581436077, |
| "task_line_acc": 0.9483362521891419, |
| "tests_acc": 0.9807355516637478, |
| "changes_acc": 0.9290718038528897, |
| "risk_acc": 0.9833625218914186, |
| "instruction_acc": 0.9912434325744308, |
| "progress_acc": 0.5788091068301225, |
| "failure_repeats_acc": 0.9168126094570929, |
| "score_acc": 0.8598949211908932, |
| "progress_within_10": 0.7145359019264448, |
| "score_within_1_anchor": 0.9754816112084063 |
| }, |
| "real": { |
| "gold": { |
| "n": 60, |
| "category_coarse": 58, |
| "category_exact": 58, |
| "gate": 57, |
| "hid": 1, |
| "surfaced": 2, |
| "misses": [ |
| "c8f4b58a8e [working/claude] got error att=0.99 want working|idle", |
| "f19b520cb2 [interrupted/codex] got working att=0.52 want working", |
| "03f3c93a3e [error/cursor] got idle att=0.28 want error" |
| ], |
| "tests": 59, |
| "changes": 50, |
| "score": 55, |
| "score_within_1_anchor": 58, |
| "task_present_agree": 24 |
| }, |
| "agents": { |
| "n": 93, |
| "category_coarse": 87, |
| "category_exact": 87, |
| "gate": 89, |
| "hid": 4, |
| "surfaced": 0, |
| "misses": [ |
| "crush#2 [finished/crush] got working att=0.41 want finished", |
| "crush#3 [question/crush] got working att=0.23 want question", |
| "goose#3 [question/goose] got approval att=1.00 want question", |
| "kiro#1 [finished/kiro] got working att=0.29 want finished", |
| "mistral#1 [question/mistral] got approval att=1.00 want question", |
| "qwen#2 [finished/qwen] got working att=0.40 want finished" |
| ] |
| }, |
| "summary_vs_luna": { |
| "summary_n": 153, |
| "summary_empty_agree": 0.7777777777777778, |
| "summary_top1_in_gold": 0.3425925925925926, |
| "summary_top1_exact": 0.3148148148148148, |
| "summary_any_hit": 0.6574074074074074, |
| "summary_precision": 0.44541484716157204, |
| "summary_recall": 0.53125, |
| "summary_f1": 0.48456057007125897, |
| "summary_lines_per_card": 1.4967320261437909 |
| }, |
| "card_vs_gold_scores": { |
| "n": 153, |
| "first_good": 0.7450980392156863, |
| "first_best": 0.6143790849673203, |
| "noise_rate": 0.21717171717171718, |
| "lines_per_card": 1.2941176470588236, |
| "empty_when_something": 0.006535947712418301 |
| }, |
| "lines_vs_gold": { |
| "n": 153, |
| "headline_acc": 0.6078431372549019, |
| "task_acc": 0.9673202614379085, |
| "options_exact": 0.782608695652174, |
| "options_false_on_none": 0.009345794392523364, |
| "summary_n": 153, |
| "summary_empty_agree": 0.8300653594771242, |
| "summary_top1_in_gold": 0.30327868852459017, |
| "summary_top1_exact": 0.2540983606557377, |
| "summary_any_hit": 0.5409836065573771, |
| "summary_precision": 0.38427947598253276, |
| "summary_recall": 0.36213991769547327, |
| "summary_f1": 0.3728813559322034, |
| "summary_lines_per_card": 1.4967320261437909, |
| "card_any_hit": 0.8206896551724138, |
| "card_precision": 0.7590361445783133, |
| "card_recall": 0.48711340206185566, |
| "card_f1": 0.5934065934065934 |
| } |
| } |
| } |
| } |
