| { |
| "recorded_date": "2026-10-02", |
| "checkpoint_sha256": "642eb7a64bfc95be17c8ed536e71a9d695eee79a13449847563f28644cf7e7ac", |
| "baseline": "development reference", |
| "candidate": "EmojiFinder", |
| "sets": { |
| "user": { |
| "before": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f05": 1.0, |
| "extra": 0, |
| "missing": 0, |
| "queries": 20, |
| "exact": 20, |
| "queries_with_extras": 0, |
| "first_ok": 20, |
| "first_total": 20, |
| "primary_order_ok": 20 |
| }, |
| "after": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f05": 1.0, |
| "extra": 0, |
| "missing": 0, |
| "queries": 20, |
| "exact": 20, |
| "queries_with_extras": 0, |
| "first_ok": 20, |
| "first_total": 20, |
| "primary_order_ok": 20 |
| }, |
| "queries": 20 |
| }, |
| "probes": { |
| "before": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f05": 1.0, |
| "extra": 0, |
| "missing": 0, |
| "queries": 191, |
| "exact": 191, |
| "queries_with_extras": 0, |
| "first_ok": 36, |
| "first_total": 36, |
| "primary_order_ok": 36 |
| }, |
| "after": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f05": 1.0, |
| "extra": 0, |
| "missing": 0, |
| "queries": 191, |
| "exact": 191, |
| "queries_with_extras": 0, |
| "first_ok": 36, |
| "first_total": 36, |
| "primary_order_ok": 36 |
| }, |
| "queries": 191 |
| }, |
| "reviewed": { |
| "before": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f05": 1.0, |
| "extra": 0, |
| "missing": 0, |
| "queries": 1839, |
| "exact": 1839, |
| "queries_with_extras": 0, |
| "first_ok": 1660, |
| "first_total": 1660, |
| "primary_order_ok": 1660 |
| }, |
| "after": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f05": 1.0, |
| "extra": 0, |
| "missing": 0, |
| "queries": 1839, |
| "exact": 1839, |
| "queries_with_extras": 0, |
| "first_ok": 1660, |
| "first_total": 1660, |
| "primary_order_ok": 1660 |
| }, |
| "queries": 1839 |
| }, |
| "validation": { |
| "before": { |
| "precision": 0.6142857142857143, |
| "recall": 0.7478260869565218, |
| "f05": 0.6370370370370371, |
| "extra": 54, |
| "missing": 29, |
| "queries": 111, |
| "exact": 68, |
| "queries_with_extras": 23, |
| "first_ok": 84, |
| "first_total": 111, |
| "primary_order_ok": 111 |
| }, |
| "after": { |
| "precision": 0.6027397260273972, |
| "recall": 0.7652173913043478, |
| "f05": 0.6294706723891274, |
| "extra": 58, |
| "missing": 27, |
| "queries": 111, |
| "exact": 67, |
| "queries_with_extras": 24, |
| "first_ok": 84, |
| "first_total": 111, |
| "primary_order_ok": 111 |
| }, |
| "queries": 111 |
| }, |
| "test": { |
| "before": { |
| "precision": 0.5562913907284768, |
| "recall": 0.717948717948718, |
| "f05": 0.5825242718446603, |
| "extra": 67, |
| "missing": 33, |
| "queries": 112, |
| "exact": 61, |
| "queries_with_extras": 24, |
| "first_ok": 74, |
| "first_total": 112, |
| "primary_order_ok": 112 |
| }, |
| "after": { |
| "precision": 0.5838926174496645, |
| "recall": 0.7435897435897436, |
| "f05": 0.6100981767180926, |
| "extra": 62, |
| "missing": 30, |
| "queries": 112, |
| "exact": 64, |
| "queries_with_extras": 22, |
| "first_ok": 81, |
| "first_total": 112, |
| "primary_order_ok": 112 |
| }, |
| "queries": 112 |
| }, |
| "old_validation": { |
| "before": { |
| "precision": 0.9981782605762094, |
| "recall": 0.9908244591788895, |
| "f05": 0.9966987805699656, |
| "extra": 27, |
| "missing": 137, |
| "queries": 3037, |
| "exact": 2965, |
| "queries_with_extras": 20, |
| "first_ok": 0, |
| "first_total": 0, |
| "primary_order_ok": 0 |
| }, |
| "after": { |
| "precision": 0.9988487844518182, |
| "recall": 0.9878775701560512, |
| "f05": 0.9966350896633738, |
| "extra": 17, |
| "missing": 181, |
| "queries": 3037, |
| "exact": 2962, |
| "queries_with_extras": 16, |
| "first_ok": 0, |
| "first_total": 0, |
| "primary_order_ok": 0 |
| }, |
| "queries": 3037 |
| }, |
| "old_test": { |
| "before": { |
| "precision": 0.9986547386829892, |
| "recall": 0.99530736743313, |
| "f05": 0.9979834644081467, |
| "extra": 20, |
| "missing": 70, |
| "queries": 3111, |
| "exact": 3059, |
| "queries_with_extras": 16, |
| "first_ok": 0, |
| "first_total": 0, |
| "primary_order_ok": 0 |
| }, |
| "after": { |
| "precision": 0.9982482145263442, |
| "recall": 0.9932292015820876, |
| "f05": 0.9972403580803662, |
| "extra": 26, |
| "missing": 101, |
| "queries": 3111, |
| "exact": 3052, |
| "queries_with_extras": 19, |
| "first_ok": 0, |
| "first_total": 0, |
| "primary_order_ok": 0 |
| }, |
| "queries": 3111 |
| }, |
| "previous_test": { |
| "before": { |
| "precision": 0.9985980804486143, |
| "recall": 0.9966634377354429, |
| "f05": 0.9982105512795636, |
| "extra": 13, |
| "missing": 31, |
| "queries": 2944, |
| "exact": 2923, |
| "queries_with_extras": 11, |
| "first_ok": 0, |
| "first_total": 0, |
| "primary_order_ok": 0 |
| }, |
| "after": { |
| "precision": 0.9991384880465216, |
| "recall": 0.9986007964697019, |
| "f05": 0.9990309034133734, |
| "extra": 8, |
| "missing": 13, |
| "queries": 2944, |
| "exact": 2930, |
| "queries_with_extras": 8, |
| "first_ok": 0, |
| "first_total": 0, |
| "primary_order_ok": 0 |
| }, |
| "queries": 2944 |
| }, |
| "previous_phrase_validation": { |
| "before": { |
| "precision": 0.9983265598852499, |
| "recall": 0.9870007090522335, |
| "f05": 0.9960406430377332, |
| "extra": 7, |
| "missing": 55, |
| "queries": 1629, |
| "exact": 1606, |
| "queries_with_extras": 6, |
| "first_ok": 1447, |
| "first_total": 1452, |
| "primary_order_ok": 1445 |
| }, |
| "after": { |
| "precision": 0.9820431933996603, |
| "recall": 0.9565114630111085, |
| "f05": 0.9768283852280956, |
| "extra": 74, |
| "missing": 184, |
| "queries": 1629, |
| "exact": 1533, |
| "queries_with_extras": 23, |
| "first_ok": 1414, |
| "first_total": 1452, |
| "primary_order_ok": 1422 |
| }, |
| "queries": 1629 |
| }, |
| "previous_phrase_test": { |
| "before": { |
| "precision": 0.9966547192353644, |
| "recall": 0.9858189553297093, |
| "f05": 0.994468551809642, |
| "extra": 14, |
| "missing": 60, |
| "queries": 1629, |
| "exact": 1592, |
| "queries_with_extras": 6, |
| "first_ok": 1446, |
| "first_total": 1452, |
| "primary_order_ok": 1444 |
| }, |
| "after": { |
| "precision": 0.9615384615384616, |
| "recall": 0.9926731269203498, |
| "f05": 0.9676081647698475, |
| "extra": 168, |
| "missing": 31, |
| "queries": 1629, |
| "exact": 1600, |
| "queries_with_extras": 22, |
| "first_ok": 1441, |
| "first_total": 1452, |
| "primary_order_ok": 1445 |
| }, |
| "queries": 1629 |
| } |
| }, |
| "categories": { |
| "train": { |
| "before": { |
| "queries": 148, |
| "passed": 9, |
| "primary_first": 85, |
| "primary_before_related": 22, |
| "required_total": 2466, |
| "missing_total": 2063, |
| "forbidden_total": 0 |
| }, |
| "after": { |
| "queries": 148, |
| "passed": 148, |
| "primary_first": 148, |
| "primary_before_related": 148, |
| "required_total": 2466, |
| "missing_total": 0, |
| "forbidden_total": 0 |
| } |
| }, |
| "validation": { |
| "before": { |
| "queries": 64, |
| "passed": 0, |
| "primary_first": 27, |
| "primary_before_related": 4, |
| "required_total": 1048, |
| "missing_total": 926, |
| "forbidden_total": 0 |
| }, |
| "after": { |
| "queries": 64, |
| "passed": 11, |
| "primary_first": 46, |
| "primary_before_related": 19, |
| "required_total": 1048, |
| "missing_total": 512, |
| "forbidden_total": 0 |
| } |
| }, |
| "test": { |
| "before": { |
| "queries": 64, |
| "passed": 0, |
| "primary_first": 24, |
| "primary_before_related": 5, |
| "required_total": 1048, |
| "missing_total": 943, |
| "forbidden_total": 0 |
| }, |
| "after": { |
| "queries": 64, |
| "passed": 0, |
| "primary_first": 32, |
| "primary_before_related": 7, |
| "required_total": 1048, |
| "missing_total": 842, |
| "forbidden_total": 0 |
| } |
| } |
| }, |
| "limitations": "Assistant-authored labels, overlapping regression suites and prior test exposure; not blind human evaluation." |
| } |
|
|
