| { |
| "task": "LEMBSummScreenFDRetrieval", |
| "split": "validation", |
| "subset": "default", |
| "main_metric": "ndcg_at_10", |
| "scores": { |
| "original": 0.76988, |
| "vela": 0.8323, |
| "r6": 0.88493, |
| "r9": 0.87688, |
| "r13": 0.88041, |
| "r20": 0.88471, |
| "candidate": 0.88362, |
| "r3": 0.86925 |
| }, |
| "score_scale": "Official native FP32 metrics on0–1 scale; multiply by100 for display.", |
| "candidate_content_sha256": "3bf3ad85cdc82f8c5e67d44a59088b89de98f465a1a2b9126ff43a5f6f4b6225", |
| "dataset_revision": "6e346642246bfb4928c560ee08640dc84d074e8c", |
| "coverage": { |
| "queries": 336, |
| "documents": 336, |
| "configured_max_length": 32768, |
| "document_token_lengths": { |
| "minimum": 2662, |
| "maximum": 23856, |
| "sum": 2887533, |
| "count": 336 |
| }, |
| "truncated_documents": 0 |
| }, |
| "paired_comparison": { |
| "status": "completed", |
| "gate_component": { |
| "passed": true, |
| "delta_vs_r13": 0.0032154935251345294, |
| "minimum_delta": -0.005, |
| "units": "raw score delta", |
| "release_decision": false |
| }, |
| "scope": "Already exposed long-development validation. No new model inference; no sealed confirmation data.", |
| "candidate_content_sha256": "3bf3ad85cdc82f8c5e67d44a59088b89de98f465a1a2b9126ff43a5f6f4b6225", |
| "task": "LEMBSummScreenFDRetrieval", |
| "split": "validation", |
| "metric": "ndcg_at_10", |
| "replicates": 2000, |
| "seed": 20260915, |
| "unit": "Paired query/source-document, verified336distinct sources", |
| "evidence": { |
| "original": { |
| "official_ndcg_at_10": 0.76988, |
| "recomputed_ndcg_at_10": 0.7698826358058495, |
| "result_sha256": "2e3c8594a8b7f7f62fad52f7dceaaf338b1146f8d90ad731b6345d50c6ffe833", |
| "prediction_sha256": "e14f41b52de7d7a9f6956283a9dee3dcdf496f42c90f824153922d04ca4ef743", |
| "qrels_sha256": "e8e521e2122fedc7d57678af6819ae09418cc7e346b6f029099da1a606c64558", |
| "token_lengths_sha256": "e8c0d697b5a37c08a9e3ed177295a6e6a23fd15e2e81adf0e17f1a4dc88f7d93" |
| }, |
| "vela": { |
| "official_ndcg_at_10": 0.8323, |
| "recomputed_ndcg_at_10": 0.8323025262730762, |
| "result_sha256": "2a0b49ad538fa71dccd5bde5bdbd0aa3d877708f7ac6cc3ea435852c7812ddf8", |
| "prediction_sha256": "9340d48309f598ab2ee5ba3b58fcfe37a530f052528ead0f754ab37bea8fa030", |
| "qrels_sha256": "e8e521e2122fedc7d57678af6819ae09418cc7e346b6f029099da1a606c64558", |
| "token_lengths_sha256": "3db940cd46ac4b4d4c62af8d98baa1b116a761305f2005e2c658e9bffe728862" |
| }, |
| "r6-intent-all22-lr1e5": { |
| "official_ndcg_at_10": 0.88493, |
| "recomputed_ndcg_at_10": 0.884925127232939, |
| "result_sha256": "b7ea70458c34d2fd0f6210d19f95440fb33cafba551f1d404978cc8ca6200241", |
| "prediction_sha256": "8cb20cd7bd7442d7c80fb29c5cf54525a111eb6e67c4b509ed90314146da79f7", |
| "qrels_sha256": "e8e521e2122fedc7d57678af6819ae09418cc7e346b6f029099da1a606c64558", |
| "token_lengths_sha256": "cdd98ac1f6110610422007186dc6b19823466446aa6cfca379b45ac4d6b89f0c" |
| }, |
| "r9-guide-mask-lr3e6": { |
| "official_ndcg_at_10": 0.87688, |
| "recomputed_ndcg_at_10": 0.876882778899065, |
| "result_sha256": "95f44d11d6ff02fd672cc2c0fe7398ee148177ffd5291277d14d305e625499b0", |
| "prediction_sha256": "7c5f26d11fcb98cb6efd07bc1f56f590b463dbda83b017867783c6cd3a29fc07", |
| "qrels_sha256": "e8e521e2122fedc7d57678af6819ae09418cc7e346b6f029099da1a606c64558", |
| "token_lengths_sha256": "0c78366b6fddb877399974ba0e3671d5f49cf14a49dc38db52126504e9ce5151" |
| }, |
| "r13-bge-kd-w01": { |
| "official_ndcg_at_10": 0.88041, |
| "recomputed_ndcg_at_10": 0.8804095047305854, |
| "result_sha256": "34afebd3836a1d25bf0e7798a6c47d979ff4537d389a0fa46494f81ba3b3572f", |
| "prediction_sha256": "9d314706274078458ea080b56c7c63a2b3e339ea20584d980bc919cfc43c2dbf", |
| "qrels_sha256": "e8e521e2122fedc7d57678af6819ae09418cc7e346b6f029099da1a606c64558", |
| "token_lengths_sha256": "fd547245c7fdc2b299dd8a4e1bd7631ccee3fb299911933301cacacd6e7ad7ef" |
| }, |
| "r20-t2ranking-zh8192": { |
| "official_ndcg_at_10": 0.88471, |
| "recomputed_ndcg_at_10": 0.8847132026337041, |
| "result_sha256": "469de4a61c0c787dfe3bfc52805fa51b6d22e94ca712682f46a5868b4f85f088", |
| "prediction_sha256": "801ab1f7fb36034b361cdc0996cfff3473ad4704292f301e0294b933566d1400", |
| "qrels_sha256": "e8e521e2122fedc7d57678af6819ae09418cc7e346b6f029099da1a606c64558", |
| "token_lengths_sha256": "2df6252ed766d521014b14ee0d9d9355c6285e7dcf7a87177daa44a3acdbf9a3" |
| }, |
| "r24-new-queries": { |
| "official_ndcg_at_10": 0.88362, |
| "recomputed_ndcg_at_10": 0.8836249982557198, |
| "result_sha256": "fe279244d0ae16d2c759e4320e28b0b3e0f82688386d5aa429103430277e27b3", |
| "prediction_sha256": "4415bf7da963741204a32979abd479066736bb47fe6dda101a6e6618abeb4b14", |
| "qrels_sha256": "e8e521e2122fedc7d57678af6819ae09418cc7e346b6f029099da1a606c64558", |
| "token_lengths_sha256": "c28375f904225f4bc5d62415e676f732970fd04ca540b45119abb0a0e4bd6800" |
| } |
| }, |
| "rows": [ |
| { |
| "candidate": "r24-new-queries", |
| "baseline": "original", |
| "group": "all", |
| "queries": 336, |
| "candidate_score": 0.8836249982557198, |
| "baseline_score": 0.7698826358058495, |
| "delta": 0.11374236244987043, |
| "paired_bootstrap_95_interval": [ |
| 0.08580597893036003, |
| 0.1436500045695695 |
| ], |
| "improved": 94, |
| "equal": 225, |
| "regressed": 17 |
| }, |
| { |
| "candidate": "r24-new-queries", |
| "baseline": "original", |
| "group": "relevant_document_at_most_8192_tokens", |
| "queries": 170, |
| "candidate_score": 0.9207402405116178, |
| "baseline_score": 0.839537458569051, |
| "delta": 0.08120278194256693, |
| "paired_bootstrap_95_interval": [ |
| 0.04316328424717932, |
| 0.12102952443348348 |
| ], |
| "improved": 35, |
| "equal": 126, |
| "regressed": 9 |
| }, |
| { |
| "candidate": "r24-new-queries", |
| "baseline": "original", |
| "group": "relevant_document_above_8192_tokens", |
| "queries": 166, |
| "candidate_score": 0.845615412812933, |
| "baseline_score": 0.6985493835784745, |
| "delta": 0.14706602923445836, |
| "paired_bootstrap_95_interval": [ |
| 0.10448118709621954, |
| 0.19084205323592268 |
| ], |
| "improved": 59, |
| "equal": 99, |
| "regressed": 8 |
| }, |
| { |
| "candidate": "r24-new-queries", |
| "baseline": "vela", |
| "group": "all", |
| "queries": 336, |
| "candidate_score": 0.8836249982557198, |
| "baseline_score": 0.8323025262730762, |
| "delta": 0.05132247198264383, |
| "paired_bootstrap_95_interval": [ |
| 0.02595699843539391, |
| 0.0778629398066223 |
| ], |
| "improved": 59, |
| "equal": 246, |
| "regressed": 31 |
| }, |
| { |
| "candidate": "r24-new-queries", |
| "baseline": "vela", |
| "group": "relevant_document_at_most_8192_tokens", |
| "queries": 170, |
| "candidate_score": 0.9207402405116178, |
| "baseline_score": 0.8468191092768992, |
| "delta": 0.07392113123471859, |
| "paired_bootstrap_95_interval": [ |
| 0.04501357914334632, |
| 0.10670490435922182 |
| ], |
| "improved": 30, |
| "equal": 135, |
| "regressed": 5 |
| }, |
| { |
| "candidate": "r24-new-queries", |
| "baseline": "vela", |
| "group": "relevant_document_above_8192_tokens", |
| "queries": 166, |
| "candidate_score": 0.845615412812933, |
| "baseline_score": 0.817436146088438, |
| "delta": 0.02817926672449499, |
| "paired_bootstrap_95_interval": [ |
| -0.014128021736724831, |
| 0.07003966358355486 |
| ], |
| "improved": 29, |
| "equal": 111, |
| "regressed": 26 |
| }, |
| { |
| "candidate": "r24-new-queries", |
| "baseline": "r6-intent-all22-lr1e5", |
| "group": "all", |
| "queries": 336, |
| "candidate_score": 0.8836249982557198, |
| "baseline_score": 0.884925127232939, |
| "delta": -0.0013001289772192758, |
| "paired_bootstrap_95_interval": [ |
| -0.008658072287171298, |
| 0.005251621338530261 |
| ], |
| "improved": 18, |
| "equal": 306, |
| "regressed": 12 |
| }, |
| { |
| "candidate": "r24-new-queries", |
| "baseline": "r6-intent-all22-lr1e5", |
| "group": "relevant_document_at_most_8192_tokens", |
| "queries": 170, |
| "candidate_score": 0.9207402405116178, |
| "baseline_score": 0.9245553524790817, |
| "delta": -0.0038151119674638572, |
| "paired_bootstrap_95_interval": [ |
| -0.012504089298200982, |
| 0.002641760875011354 |
| ], |
| "improved": 4, |
| "equal": 162, |
| "regressed": 4 |
| }, |
| { |
| "candidate": "r24-new-queries", |
| "baseline": "r6-intent-all22-lr1e5", |
| "group": "relevant_document_above_8192_tokens", |
| "queries": 166, |
| "candidate_score": 0.845615412812933, |
| "baseline_score": 0.8443399568001426, |
| "delta": 0.0012754560127902356, |
| "paired_bootstrap_95_interval": [ |
| -0.009602981539440099, |
| 0.012731905072227398 |
| ], |
| "improved": 14, |
| "equal": 144, |
| "regressed": 8 |
| }, |
| { |
| "candidate": "r24-new-queries", |
| "baseline": "r9-guide-mask-lr3e6", |
| "group": "all", |
| "queries": 336, |
| "candidate_score": 0.8836249982557198, |
| "baseline_score": 0.876882778899065, |
| "delta": 0.0067422193566549615, |
| "paired_bootstrap_95_interval": [ |
| 0.0006376803680880756, |
| 0.01385482075292914 |
| ], |
| "improved": 19, |
| "equal": 311, |
| "regressed": 6 |
| }, |
| { |
| "candidate": "r24-new-queries", |
| "baseline": "r9-guide-mask-lr3e6", |
| "group": "relevant_document_at_most_8192_tokens", |
| "queries": 170, |
| "candidate_score": 0.9207402405116178, |
| "baseline_score": 0.9142852803660881, |
| "delta": 0.006454960145529764, |
| "paired_bootstrap_95_interval": [ |
| -0.0024266134694441643, |
| 0.01719387898700963 |
| ], |
| "improved": 6, |
| "equal": 162, |
| "regressed": 2 |
| }, |
| { |
| "candidate": "r24-new-queries", |
| "baseline": "r9-guide-mask-lr3e6", |
| "group": "relevant_document_above_8192_tokens", |
| "queries": 166, |
| "candidate_score": 0.845615412812933, |
| "baseline_score": 0.838579012336451, |
| "delta": 0.00703640047648197, |
| "paired_bootstrap_95_interval": [ |
| -0.0023528919316009025, |
| 0.016606970299408556 |
| ], |
| "improved": 13, |
| "equal": 149, |
| "regressed": 4 |
| }, |
| { |
| "candidate": "r24-new-queries", |
| "baseline": "r13-bge-kd-w01", |
| "group": "all", |
| "queries": 336, |
| "candidate_score": 0.8836249982557198, |
| "baseline_score": 0.8804095047305854, |
| "delta": 0.0032154935251345294, |
| "paired_bootstrap_95_interval": [ |
| -0.002654427117917938, |
| 0.009620371358263215 |
| ], |
| "improved": 16, |
| "equal": 311, |
| "regressed": 9 |
| }, |
| { |
| "candidate": "r24-new-queries", |
| "baseline": "r13-bge-kd-w01", |
| "group": "relevant_document_at_most_8192_tokens", |
| "queries": 170, |
| "candidate_score": 0.9207402405116178, |
| "baseline_score": 0.9174067251903617, |
| "delta": 0.00333351532125612, |
| "paired_bootstrap_95_interval": [ |
| -0.004205604809041235, |
| 0.012017521119574765 |
| ], |
| "improved": 5, |
| "equal": 162, |
| "regressed": 3 |
| }, |
| { |
| "candidate": "r24-new-queries", |
| "baseline": "r13-bge-kd-w01", |
| "group": "relevant_document_above_8192_tokens", |
| "queries": 166, |
| "candidate_score": 0.845615412812933, |
| "baseline_score": 0.8425207849826216, |
| "delta": 0.003094627830311214, |
| "paired_bootstrap_95_interval": [ |
| -0.006415175063604415, |
| 0.012801972665767698 |
| ], |
| "improved": 11, |
| "equal": 149, |
| "regressed": 6 |
| }, |
| { |
| "candidate": "r24-new-queries", |
| "baseline": "r20-t2ranking-zh8192", |
| "group": "all", |
| "queries": 336, |
| "candidate_score": 0.8836249982557198, |
| "baseline_score": 0.8847132026337041, |
| "delta": -0.0010882043779841832, |
| "paired_bootstrap_95_interval": [ |
| -0.005630972134979018, |
| 0.003024722199180693 |
| ], |
| "improved": 2, |
| "equal": 326, |
| "regressed": 8 |
| }, |
| { |
| "candidate": "r24-new-queries", |
| "baseline": "r20-t2ranking-zh8192", |
| "group": "relevant_document_at_most_8192_tokens", |
| "queries": 170, |
| "candidate_score": 0.9207402405116178, |
| "baseline_score": 0.9214058122337676, |
| "delta": -0.0006655717221497551, |
| "paired_bootstrap_95_interval": [ |
| -0.00717857607088874, |
| 0.00584743262658923 |
| ], |
| "improved": 1, |
| "equal": 166, |
| "regressed": 3 |
| }, |
| { |
| "candidate": "r24-new-queries", |
| "baseline": "r20-t2ranking-zh8192", |
| "group": "relevant_document_above_8192_tokens", |
| "queries": 166, |
| "candidate_score": 0.845615412812933, |
| "baseline_score": 0.8471364337661693, |
| "delta": -0.001521020953236308, |
| "paired_bootstrap_95_interval": [ |
| -0.007675908408223043, |
| 0.004130748431944928 |
| ], |
| "improved": 1, |
| "equal": 160, |
| "regressed": 5 |
| } |
| ], |
| "r20_reference_content_sha256": "dbbba2a09857723e44f77f6084a22cbc5e49b6b281d95eaa545abbe546edde04", |
| "r20_reference_manifest_sha256": "99e03e6512f7d77defc4f43157822f3545742cd677824474f2bf2fcd3d3fafa6", |
| "runner_sha256": "5c7b45cbfb3619db1eff32b6232eb87a990eae6eb850670946f7f0e5607b20d6", |
| "launch_manifest_sha256": "7c2606f42cab34d637d58d3235dafc4b3290154d818ce4daddd3b078e4fe3688", |
| "lengths": { |
| "minimum": 2662, |
| "maximum": 23856, |
| "sum": 2887533, |
| "count": 336 |
| }, |
| "limitations": [ |
| "Conditional development-query uncertainty; fixed corpus retained.", |
| "Intervals are exploratory and not adjusted for task/subgroup multiplicity.", |
| "A single long retrieval task does not establish all long-context or 32768-token endpoint capabilities." |
| ] |
| }, |
| "scope": "Previously exposed development validation, not fresh confirmation. Native FP32 MATH SDPA, not ONNX benchmark scores." |
| } |
|
|
