{ "schema_version": 2, "updated_at": "2026-08-21T07:53:39.332214+00:00", "description": "BrainAPI benchmark results. Top published scores across suites. Updated when a suite evaluate/report completes successfully.", "benchmarks": { "locomo": { "name": "LoCoMo", "leaderboard": [ { "run_id": "locomo-compose-sota-conv26-v4d", "track": "sota", "scope": "conv-26", "samples": [ "conv-26" ], "n_non_adversarial": 152, "headline_judge_accuracy": 0.9539473684210527, "headline_judge_accuracy_pct": 95.39, "answerable_rate": 0.9407894736842105, "answer_model": "deepseek-v4-flash", "judge_model": "deepseek-v4-flash", "bench_profile": "sota", "brain": "locomoconv26nostorm", "git_sha": "78cbbae50ac37777ce0315f85156f7812b28248c", "report_path": "runs/locomo-compose-sota-conv26-v4d/report.json", "recorded_at": "2026-07-31T13:14:01.363020+00:00" }, { "run_id": "locomo-compose-sota-conv26-v4c", "track": "sota", "scope": "conv-26", "samples": [ "conv-26" ], "n_non_adversarial": 152, "headline_judge_accuracy": 0.9276315789473685, "headline_judge_accuracy_pct": 92.76, "answerable_rate": 0.9407894736842105, "answer_model": "deepseek-v4-flash", "judge_model": "deepseek-v4-flash", "bench_profile": "sota", "brain": "locomoconv26nostorm", "git_sha": "78cbbae50ac37777ce0315f85156f7812b28248c", "report_path": "runs/locomo-compose-sota-conv26-v4c/report.json", "recorded_at": "2026-07-31T13:07:26.775227+00:00" }, { "run_id": "locomo-compose-sota-conv26-v2", "track": "sota", "scope": "conv-26", "samples": [ "conv-26" ], "n_non_adversarial": 152, "headline_judge_accuracy": 0.9210526315789473, "headline_judge_accuracy_pct": 92.11, "answerable_rate": 0.9407894736842105, "answer_model": "deepseek-v4-flash", "judge_model": "deepseek-v4-flash", "bench_profile": "sota", "brain": "locomoconv26nostorm", "git_sha": "78cbbae50ac37777ce0315f85156f7812b28248c", "report_path": "runs/locomo-compose-sota-conv26-v2/report.json", "recorded_at": "2026-07-30T20:01:08.915975+00:00" }, { "run_id": "locomo-compose-sota-conv26-v4", "track": "sota", "scope": "conv-26", "samples": [ "conv-26" ], "n_non_adversarial": 152, "headline_judge_accuracy": 0.9210526315789473, "headline_judge_accuracy_pct": 92.11, "answerable_rate": 0.9407894736842105, "answer_model": "deepseek-v4-flash", "judge_model": "deepseek-v4-flash", "bench_profile": "sota", "brain": "locomoconv26nostorm", "git_sha": "78cbbae50ac37777ce0315f85156f7812b28248c", "report_path": "runs/locomo-compose-sota-conv26-v4/report.json", "recorded_at": "2026-07-31T11:28:19.452764+00:00" }, { "run_id": "locomo-compose-sota-conv26-v3", "track": "sota", "scope": "conv-26", "samples": [ "conv-26" ], "n_non_adversarial": 152, "headline_judge_accuracy": 0.9013157894736842, "headline_judge_accuracy_pct": 90.13, "answerable_rate": 0.9407894736842105, "answer_model": "deepseek-v4-flash", "judge_model": "deepseek-v4-flash", "bench_profile": "sota", "brain": "locomoconv26nostorm", "git_sha": "78cbbae50ac37777ce0315f85156f7812b28248c", "report_path": "runs/locomo-compose-sota-conv26-v3/report.json", "recorded_at": "2026-07-31T10:29:50.328215+00:00" }, { "run_id": "locomo-conv26-beamreg-product", "track": "product", "scope": "conv-26", "samples": [ "conv-26" ], "n_non_adversarial": 152, "headline_judge_accuracy": 0.9013157894736842, "headline_judge_accuracy_pct": 90.13, "answerable_rate": 0.9473684210526315, "answer_model": "deepseek-v4-flash", "judge_model": "deepseek-v4-flash", "bench_profile": "product", "brain": "locomoconv26", "git_sha": "78cbbae50ac37777ce0315f85156f7812b28248c", "report_path": "runs/locomo-conv26-beamreg-product/report.json", "recorded_at": "2026-08-01T01:58:34.693097+00:00" }, { "run_id": "locomo-compose-sota-conv26", "track": "sota", "scope": "conv-26", "samples": [ "conv-26" ], "n_non_adversarial": 152, "headline_judge_accuracy": 0.8947368421052632, "headline_judge_accuracy_pct": 89.47, "answerable_rate": 0.9407894736842105, "answer_model": "deepseek-v4-flash", "judge_model": "deepseek-v4-flash", "bench_profile": "sota", "brain": "locomoconv26nostorm", "git_sha": "78cbbae50ac37777ce0315f85156f7812b28248c", "report_path": "runs/locomo-compose-sota-conv26/report.json", "recorded_at": "2026-07-30T19:18:56.740730+00:00" }, { "run_id": "locomo-compose-sota-conv26-v4b", "track": "sota", "scope": "conv-26", "samples": [ "conv-26" ], "n_non_adversarial": 152, "headline_judge_accuracy": 0.875, "headline_judge_accuracy_pct": 87.5, "answerable_rate": 0.9407894736842105, "answer_model": "deepseek-v4-flash", "judge_model": "deepseek-v4-flash", "bench_profile": "sota", "brain": "locomoconv26nostorm", "git_sha": "78cbbae50ac37777ce0315f85156f7812b28248c", "report_path": "runs/locomo-compose-sota-conv26-v4b/report.json", "recorded_at": "2026-07-31T12:26:52.469668+00:00" }, { "run_id": "locomo-cat3fix-sota-conv26", "track": "sota", "scope": "conv-26", "samples": [ "conv-26" ], "n_non_adversarial": 152, "headline_judge_accuracy": 0.868421052631579, "headline_judge_accuracy_pct": 86.84, "answerable_rate": 0.9407894736842105, "answer_model": "deepseek-v4-flash", "judge_model": "deepseek-v4-flash", "bench_profile": "sota", "brain": "locomoconv26nostorm", "git_sha": "78cbbae50ac37777ce0315f85156f7812b28248c", "report_path": "runs/locomo-cat3fix-sota-conv26/report.json", "recorded_at": "2026-07-30T18:26:15.214852+00:00" }, { "run_id": "oldbrain-recheck-a", "track": "product", "scope": "conv-26", "samples": [ "conv-26" ], "n_non_adversarial": 152, "headline_judge_accuracy": 0.8552631578947368, "headline_judge_accuracy_pct": 85.53, "answerable_rate": 0.9473684210526315, "answer_model": "deepseek-v4-flash", "judge_model": "deepseek-v4-flash", "bench_profile": "product", "brain": "locomoconv26", "git_sha": "649f5e569de8894c05eec5f1f423024be5e9b6a4", "report_path": "runs/oldbrain-recheck-a/report.json", "recorded_at": "2026-07-28T22:05:39.768798+00:00" }, { "run_id": "locomo-batch-nostorm-cfgb", "track": "product", "scope": "conv-26", "samples": [ "conv-26" ], "n_non_adversarial": 152, "headline_judge_accuracy": 0.8552631578947368, "headline_judge_accuracy_pct": 85.53, "answerable_rate": 0.9407894736842105, "answer_model": "deepseek-v4-flash", "judge_model": "deepseek-v4-flash", "bench_profile": "product", "brain": "locomoconv26nostorm", "git_sha": "78cbbae50ac37777ce0315f85156f7812b28248c", "report_path": "runs/locomo-batch-nostorm-cfgb/report.json", "recorded_at": "2026-07-30T17:03:35.449258+00:00" }, { "run_id": "phase-sota-d1-conv26", "track": "sota", "scope": "conv-26", "samples": [ "conv-26" ], "n_non_adversarial": 152, "headline_judge_accuracy": 0.8486842105263158, "headline_judge_accuracy_pct": 84.87, "answerable_rate": 0.9407894736842105, "answer_model": "deepseek-v4-flash", "judge_model": "deepseek-v4-flash", "bench_profile": "sota", "brain": "locomoconv26clean", "git_sha": "649f5e569de8894c05eec5f1f423024be5e9b6a4", "report_path": "runs/phase-sota-d1-conv26/report.json", "recorded_at": "2026-07-29T18:51:58.400341+00:00" }, { "run_id": "locomo-nostorm-sota-conv26", "track": "sota", "scope": "conv-26", "samples": [ "conv-26" ], "n_non_adversarial": 152, "headline_judge_accuracy": 0.8486842105263158, "headline_judge_accuracy_pct": 84.87, "answerable_rate": 0.9407894736842105, "answer_model": "deepseek-v4-flash", "judge_model": "deepseek-v4-flash", "bench_profile": "sota", "brain": "locomoconv26nostorm", "git_sha": "78cbbae50ac37777ce0315f85156f7812b28248c", "report_path": "runs/locomo-nostorm-sota-conv26/report.json", "recorded_at": "2026-07-30T17:41:32.563092+00:00" }, { "run_id": "locomo-batch-cfgb", "track": "product", "scope": "conv-26", "samples": [ "conv-26" ], "n_non_adversarial": 152, "headline_judge_accuracy": 0.8421052631578947, "headline_judge_accuracy_pct": 84.21, "answerable_rate": 0.9407894736842105, "answer_model": "deepseek-v4-flash", "judge_model": "deepseek-v4-flash", "bench_profile": "product", "brain": "locomoconv26batch", "git_sha": "78cbbae50ac37777ce0315f85156f7812b28248c", "report_path": "runs/locomo-batch-cfgb/report.json", "recorded_at": "2026-07-30T15:23:27.730891+00:00" }, { "run_id": "clean-brain-eval-a", "track": "product", "scope": "conv-26", "samples": [ "conv-26" ], "n_non_adversarial": 152, "headline_judge_accuracy": 0.8355263157894737, "headline_judge_accuracy_pct": 83.55, "answerable_rate": 0.9539473684210527, "answer_model": "deepseek-v4-flash", "judge_model": "deepseek-v4-flash", "bench_profile": "product", "brain": "locomoconv26clean", "git_sha": "649f5e569de8894c05eec5f1f423024be5e9b6a4", "report_path": "runs/clean-brain-eval-a/report.json", "recorded_at": "2026-07-28T21:29:30.713806+00:00" }, { "run_id": "graph-stable-b", "track": "product", "scope": "conv-26", "samples": [ "conv-26" ], "n_non_adversarial": 152, "headline_judge_accuracy": 0.8355263157894737, "headline_judge_accuracy_pct": 83.55, "answerable_rate": 0.9671052631578947, "answer_model": "deepseek-v4-flash", "judge_model": "deepseek-v4-flash", "bench_profile": "product", "brain": "locomoconv26clean", "git_sha": "649f5e569de8894c05eec5f1f423024be5e9b6a4", "report_path": "runs/graph-stable-b/report.json", "recorded_at": "2026-07-29T10:59:22.348091+00:00" }, { "run_id": "clean-brain-a", "track": "product", "scope": "conv-26", "samples": [ "conv-26" ], "n_non_adversarial": 152, "headline_judge_accuracy": 0.8289473684210527, "headline_judge_accuracy_pct": 82.89, "answerable_rate": 0.9473684210526315, "answer_model": "deepseek-v4-flash", "judge_model": "deepseek-v4-flash", "bench_profile": "product", "brain": "locomoconv26clean", "git_sha": "649f5e569de8894c05eec5f1f423024be5e9b6a4", "report_path": "runs/clean-brain-a/report.json", "recorded_at": "2026-07-28T21:57:06.650866+00:00" }, { "run_id": "clean-brain-b", "track": "product", "scope": "conv-26", "samples": [ "conv-26" ], "n_non_adversarial": 152, "headline_judge_accuracy": 0.8289473684210527, "headline_judge_accuracy_pct": 82.89, "answerable_rate": 0.9473684210526315, "answer_model": "deepseek-v4-flash", "judge_model": "deepseek-v4-flash", "bench_profile": "product", "brain": "locomoconv26clean", "git_sha": "649f5e569de8894c05eec5f1f423024be5e9b6a4", "report_path": "runs/clean-brain-b/report.json", "recorded_at": "2026-07-28T22:13:25.651713+00:00" }, { "run_id": "baseline-clean-a", "track": "product", "scope": "conv-26", "samples": [ "conv-26" ], "n_non_adversarial": 152, "headline_judge_accuracy": 0.8223684210526315, "headline_judge_accuracy_pct": 82.24, "answerable_rate": 0.9473684210526315, "answer_model": "deepseek-v4-flash", "judge_model": "deepseek-v4-flash", "bench_profile": "product", "brain": "locomoconv26", "git_sha": "649f5e569de8894c05eec5f1f423024be5e9b6a4", "report_path": "runs/baseline-clean-a/report.json", "recorded_at": "2026-07-28T18:18:19.581660+00:00" }, { "run_id": "sota-locomo10-batch-compose", "track": "sota", "scope": "full-locomo10", "samples": [ "conv-26", "conv-30", "conv-41", "conv-42", "conv-43", "conv-44", "conv-47", "conv-48", "conv-49", "conv-50" ], "n_non_adversarial": 1540, "headline_judge_accuracy": 0.8097402597402598, "headline_judge_accuracy_pct": 80.97, "answerable_rate": 0.9363636363636364, "answer_model": "deepseek-v4-flash", "judge_model": "deepseek-v4-flash", "bench_profile": "sota", "brain": "locomof10c*", "git_sha": "78cbbae50ac37777ce0315f85156f7812b28248c", "report_path": "runs/sota-locomo10-batch-compose/report.json", "recorded_at": "2026-07-31T13:35:58.561407+00:00" }, { "run_id": "phase-d1-paths-a", "track": "product", "scope": "conv-26", "samples": [ "conv-26" ], "n_non_adversarial": 152, "headline_judge_accuracy": 0.8092105263157895, "headline_judge_accuracy_pct": 80.92, "answerable_rate": 0.9671052631578947, "answer_model": "deepseek-v4-flash", "judge_model": "deepseek-v4-flash", "bench_profile": "product", "brain": "locomoconv26clean", "git_sha": "649f5e569de8894c05eec5f1f423024be5e9b6a4", "report_path": "runs/phase-d1-paths-a/report.json", "recorded_at": "2026-07-29T16:26:21.919353+00:00" }, { "run_id": "locomo-batch-score", "track": "product", "scope": "conv-26", "samples": [ "conv-26" ], "n_non_adversarial": 152, "headline_judge_accuracy": 0.8026315789473685, "headline_judge_accuracy_pct": 80.26, "answerable_rate": 0.9407894736842105, "answer_model": "deepseek-v4-flash", "judge_model": "deepseek-v4-flash", "bench_profile": "product", "brain": "locomoconv26batch", "git_sha": "78cbbae50ac37777ce0315f85156f7812b28248c", "report_path": "runs/locomo-batch-score/report.json", "recorded_at": "2026-07-30T14:59:39.743019+00:00" } ] }, "longmemeval": { "name": "LongMemEval", "leaderboard": [] }, "beam": { "name": "BEAM", "leaderboard": [ { "run_id": "beam-1m-1-product-v3c-compose", "track": "product", "bench_profile": "product", "scope": "1M/1", "size": "1M", "samples": [ "1M/1" ], "n_questions": 20, "headline_score": 0.7897396010333467, "headline_score_pct": 78.97, "per_ability": { "abstention": 0.5, "contradiction_resolution": 0.6875, "event_ordering": 0.42170156588902286, "information_extraction": 1.0, "instruction_following": 1.0, "knowledge_update": 1.0, "multi_session_reasoning": 0.9375, "preference_following": 0.875, "summarization": 0.4756944444444444, "temporal_reasoning": 1.0 }, "answer_model": "deepseek-v4-flash", "judge_model": "deepseek-v4-flash", "brain": "beam1m1clean", "git_sha": "78cbbae50ac37777ce0315f85156f7812b28248c", "report_path": "runs/beam-1m-1-product-v3c-compose/report.json", "recorded_at": "2026-08-03T09:15:33.803818+00:00" }, { "run_id": "beam-100k-1-clean-ordv1", "track": "product", "bench_profile": "product", "scope": "100K/1", "size": "100K", "samples": [ "100K/1" ], "n_questions": 20, "headline_score": 0.7887500000000001, "headline_score_pct": 78.88, "per_ability": { "abstention": 1.0, "contradiction_resolution": 0.625, "event_ordering": 0.7875, "information_extraction": 1.0, "instruction_following": 0.75, "knowledge_update": 1.0, "multi_session_reasoning": 0.625, "preference_following": 1.0, "summarization": 0.6000000000000001, "temporal_reasoning": 0.5 }, "answer_model": "deepseek-v4-flash", "judge_model": "deepseek-v4-flash", "brain": "beam100k1clean", "git_sha": "78cbbae50ac37777ce0315f85156f7812b28248c", "report_path": "runs/beam-100k-1-clean-ordv1/report.json", "recorded_at": "2026-08-01T02:00:50.310070+00:00" }, { "run_id": "beam-100k-1-clean-product", "track": "product", "bench_profile": "product", "scope": "100K/1", "size": "100K", "samples": [ "100K/1" ], "n_questions": 20, "headline_score": 0.7398223304703363, "headline_score_pct": 73.98, "per_ability": { "abstention": 0.5, "contradiction_resolution": 0.6875, "event_ordering": 0.1357233047033631, "information_extraction": 0.75, "instruction_following": 1.0, "knowledge_update": 1.0, "multi_session_reasoning": 0.875, "preference_following": 1.0, "summarization": 0.7, "temporal_reasoning": 0.75 }, "answer_model": "deepseek-v4-flash", "judge_model": "deepseek-v4-flash", "brain": "beam100k1clean", "git_sha": "78cbbae50ac37777ce0315f85156f7812b28248c", "report_path": "runs/beam-100k-1-clean-product/report.json", "recorded_at": "2026-08-01T02:00:51.059632+00:00" }, { "run_id": "beam-1m-1-product-v2", "track": "product", "bench_profile": "product", "scope": "1M/1", "size": "1M", "samples": [ "1M/1" ], "n_questions": 20, "headline_score": 0.7014047619826247, "headline_score_pct": 70.14, "per_ability": { "abstention": 0.5, "contradiction_resolution": 0.4375, "event_ordering": 0.2883531753818027, "information_extraction": 1.0, "instruction_following": 1.0, "knowledge_update": 1.0, "multi_session_reasoning": 0.625, "preference_following": 0.75, "summarization": 0.4131944444444444, "temporal_reasoning": 1.0 }, "answer_model": "deepseek-v4-flash", "judge_model": "deepseek-v4-flash", "brain": "beam1m1clean", "git_sha": "78cbbae50ac37777ce0315f85156f7812b28248c", "report_path": "runs/beam-1m-1-product-v2/report.json", "recorded_at": "2026-08-03T07:58:27.006989+00:00" }, { "run_id": "beam-1m-1-product-v3b", "track": "product", "bench_profile": "product", "scope": "1M/1", "size": "1M", "samples": [ "1M/1" ], "n_questions": 20, "headline_score": 0.6497747205349994, "headline_score_pct": 64.98, "per_ability": { "abstention": 0.5, "contradiction_resolution": 0.5625, "event_ordering": 0.4213583164611052, "information_extraction": 1.0, "instruction_following": 1.0, "knowledge_update": 0.5, "multi_session_reasoning": 0.75, "preference_following": 0.875, "summarization": 0.3888888888888889, "temporal_reasoning": 0.5 }, "answer_model": "deepseek-v4-flash", "judge_model": "deepseek-v4-flash", "brain": "beam1m1clean", "git_sha": "78cbbae50ac37777ce0315f85156f7812b28248c", "report_path": "runs/beam-1m-1-product-v3b/report.json", "recorded_at": "2026-08-03T09:14:47.079177+00:00" }, { "run_id": "beam-100k-1-sota", "track": "sota", "bench_profile": "sota", "scope": "100K/1", "size": "100K", "samples": [ "100K/1" ], "n_questions": 20, "headline_score": 0.6483928571428572, "headline_score_pct": 64.84, "per_ability": { "abstention": 0.0, "contradiction_resolution": 0.625, "event_ordering": 0.1339285714285714, "information_extraction": 0.75, "instruction_following": 0.75, "knowledge_update": 1.0, "multi_session_reasoning": 0.875, "preference_following": 1.0, "summarization": 0.6, "temporal_reasoning": 0.75 }, "answer_model": "deepseek-v4-flash", "judge_model": "deepseek-v4-flash", "brain": "beam100k1", "git_sha": "78cbbae50ac37777ce0315f85156f7812b28248c", "report_path": "runs/beam-100k-1-sota/report.json", "recorded_at": "2026-08-01T02:01:11.568901+00:00" }, { "run_id": "beam-100k-1-clean-sota", "track": "sota", "bench_profile": "sota", "scope": "100K/1", "size": "100K", "samples": [ "100K/1" ], "n_questions": 20, "headline_score": 0.6245833333333334, "headline_score_pct": 62.46, "per_ability": { "abstention": 0.0, "contradiction_resolution": 0.875, "event_ordering": 0.1458333333333333, "information_extraction": 0.5, "instruction_following": 0.75, "knowledge_update": 1.0, "multi_session_reasoning": 0.625, "preference_following": 1.0, "summarization": 0.6000000000000001, "temporal_reasoning": 0.75 }, "answer_model": "deepseek-v4-flash", "judge_model": "deepseek-v4-flash", "brain": "beam100k1clean", "git_sha": "78cbbae50ac37777ce0315f85156f7812b28248c", "report_path": "runs/beam-100k-1-clean-sota/report.json", "recorded_at": "2026-08-01T02:01:12.316230+00:00" }, { "run_id": "beam-100k-1-product-prompts", "track": "product", "bench_profile": "product", "scope": "100K/1", "size": "100K", "samples": [ "100K/1" ], "n_questions": 20, "headline_score": 0.6042261904761904, "headline_score_pct": 60.42, "per_ability": { "abstention": 0.0, "contradiction_resolution": 0.5, "event_ordering": 0.1339285714285714, "information_extraction": 0.9583333333333333, "instruction_following": 0.75, "knowledge_update": 1.0, "multi_session_reasoning": 0.75, "preference_following": 0.75, "summarization": 0.7, "temporal_reasoning": 0.5 }, "answer_model": "deepseek-v4-flash", "judge_model": "deepseek-v4-flash", "brain": "beam100k1", "git_sha": "78cbbae50ac37777ce0315f85156f7812b28248c", "report_path": "runs/beam-100k-1-product-prompts/report.json", "recorded_at": "2026-08-01T02:01:13.074254+00:00" }, { "run_id": "beam-100k-1-configb", "track": "product", "bench_profile": "product", "scope": "100K/1", "size": "100K", "samples": [ "100K/1" ], "n_questions": 20, "headline_score": 0.559333376188726, "headline_score_pct": 55.93, "per_ability": { "abstention": 0.25, "contradiction_resolution": 0.5, "event_ordering": 0.14333376188726035, "information_extraction": 0.5, "instruction_following": 0.75, "knowledge_update": 1.0, "multi_session_reasoning": 0.625, "preference_following": 0.75, "summarization": 0.7, "temporal_reasoning": 0.375 }, "answer_model": "deepseek-v4-flash", "judge_model": "deepseek-v4-flash", "brain": "beam100k1", "git_sha": "78cbbae50ac37777ce0315f85156f7812b28248c", "report_path": "runs/beam-100k-1-configb/report.json", "recorded_at": "2026-08-01T02:01:13.832770+00:00" }, { "run_id": "beam-100k-1", "track": "product", "bench_profile": "product", "scope": "100K/1", "size": "100K", "samples": [ "100K/1" ], "n_questions": 20, "headline_score": 0.5558928571428572, "headline_score_pct": 55.59, "per_ability": { "abstention": 0.5, "contradiction_resolution": 0.5, "event_ordering": 0.1339285714285714, "information_extraction": 0.5, "instruction_following": 0.5, "knowledge_update": 1.0, "multi_session_reasoning": 0.625, "preference_following": 0.75, "summarization": 0.55, "temporal_reasoning": 0.5 }, "answer_model": "deepseek-v4-flash", "judge_model": "deepseek-v4-flash", "brain": "beam100k1", "git_sha": "78cbbae50ac37777ce0315f85156f7812b28248c", "report_path": "runs/beam-100k-1/report.json", "recorded_at": "2026-08-01T02:01:14.597463+00:00" }, { "run_id": "beam-1m-1-clean-product", "track": "product", "bench_profile": "product", "scope": "1M/1", "size": "1M", "samples": [ "1M/1" ], "n_questions": 20, "headline_score": 0.5520385817869101, "headline_score_pct": 55.2, "per_ability": { "abstention": 0.5, "contradiction_resolution": 0.375, "event_ordering": 0.35024692898021254, "information_extraction": 1.0, "instruction_following": 0.5, "knowledge_update": 0.5, "multi_session_reasoning": 0.625, "preference_following": 0.75, "summarization": 0.4201388888888889, "temporal_reasoning": 0.5 }, "answer_model": "deepseek-v4-flash", "judge_model": "deepseek-v4-flash", "brain": "beam1m1clean", "git_sha": "78cbbae50ac37777ce0315f85156f7812b28248c", "report_path": "runs/beam-1m-1-clean-product/report.json", "recorded_at": "2026-08-03T06:59:14.016975+00:00" } ] }, "recsys": { "name": "RecSys (event-KG recommend)", "leaderboard": [ { "run_id": "recsys-20260805T200653Z", "brain": "demorecsys", "n_users": 20, "hit_rate@10": 0.1, "hit_rate@20": 0.1, "recall@10": 0.1, "recall@20": 0.1, "mean_intra_list_diversity": 0.20536531279178338, "mean_type_coverage": 2.0, "git_sha": "cf61c99b809730cc9eb35db3532e018bdd038a55", "report_path": "runs/recsys-20260805T200653Z/report.json", "recorded_at": "2026-08-05T20:07:44.707984+00:00" } ] }, "search": { "name": "Search (hybrid BM25 + dense)", "representative_run_id": "search-wands-passages-k50", "representative_scope": "WANDS frozen slice: 66 queries / 2000 documents", "leaderboard": [ { "run_id": "search-20260818T111250Z", "brain": "searchbenchsmoke", "dataset": "search_toy.jsonl", "fusion": "rrf", "rerank": "none", "n_queries": 10, "ndcg@10": 1.0, "recall@10": 1.0, "mrr": 1.0, "p50_retrieve_ms": 25.1575, "p95_retrieve_ms": 27.67875, "git_sha": "f42c4a8eaadb3c0cc79f78c34b0e1a70534bee10", "report_path": "runs/search-20260818T111250Z/report.json", "recorded_at": "2026-08-18T11:13:57.784280+00:00" }, { "run_id": "search-toy-cc", "brain": "searchbenchsmoke", "dataset": "search_toy.jsonl", "fusion": "cc", "rerank": "none", "n_queries": 10, "ndcg@10": 1.0, "recall@10": 1.0, "mrr": 1.0, "p50_retrieve_ms": 24.2875, "p95_retrieve_ms": 41.931949999999986, "git_sha": "f42c4a8eaadb3c0cc79f78c34b0e1a70534bee10", "report_path": "runs/search-toy-cc/report.json", "recorded_at": "2026-08-18T11:15:19.039999+00:00" }, { "run_id": "search-esci-slice-allcols", "brain": "searchbenchesci20", "dataset": "search_esci_slice.jsonl", "fusion": "rrf", "rerank": "none", "channels": [ "passages", "entities", "communities" ], "n_queries": 11, "ndcg@10": 0.6815199154995802, "recall@10": 0.4656565656565656, "mrr": 0.8484848484848485, "p50_retrieve_ms": 108.159, "p95_retrieve_ms": 129.32150000000001, "git_sha": "f42c4a8eaadb3c0cc79f78c34b0e1a70534bee10", "report_path": "runs/search-esci-slice-allcols/report.json", "recorded_at": "2026-08-18T13:02:36.140118+00:00" }, { "run_id": "search-esci-slice-passages", "brain": "searchbenchesci20", "dataset": "search_esci_slice.jsonl", "fusion": "rrf", "rerank": "none", "channels": [ "passages" ], "n_queries": 11, "ndcg@10": 0.7579469533596556, "recall@10": 0.5106060606060606, "mrr": 0.8484848484848485, "p50_retrieve_ms": 29.279, "p95_retrieve_ms": 153.97, "git_sha": "f42c4a8eaadb3c0cc79f78c34b0e1a70534bee10", "report_path": "runs/search-esci-slice-passages/report.json", "recorded_at": "2026-08-18T13:02:47.533040+00:00" }, { "run_id": "search-esci-slice-entities", "brain": "searchbenchesci20", "dataset": "search_esci_slice.jsonl", "fusion": "rrf", "rerank": "none", "channels": [ "entities" ], "n_queries": 11, "ndcg@10": 0.5323459350114446, "recall@10": 0.4151515151515151, "mrr": 0.6673553719008264, "p50_retrieve_ms": 52.75, "p95_retrieve_ms": 75.479, "git_sha": "f42c4a8eaadb3c0cc79f78c34b0e1a70534bee10", "report_path": "runs/search-esci-slice-entities/report.json", "recorded_at": "2026-08-18T13:31:39.862006+00:00" }, { "run_id": "search-esci-slice-communities", "brain": "searchbenchesci20", "dataset": "search_esci_slice.jsonl", "fusion": "rrf", "rerank": "none", "channels": [ "communities" ], "n_queries": 11, "ndcg@10": 0.0, "recall@10": 0.0, "mrr": 0.0, "p50_retrieve_ms": 51.536, "p95_retrieve_ms": 58.432500000000005, "git_sha": "f42c4a8eaadb3c0cc79f78c34b0e1a70534bee10", "report_path": "runs/search-esci-slice-communities/report.json", "recorded_at": "2026-08-18T13:31:42.972981+00:00" }, { "run_id": "search-esci-slice-pe", "brain": "searchbenchesci20", "dataset": "search_esci_slice.jsonl", "fusion": "rrf", "rerank": "none", "channels": [ "passages", "entities" ], "n_queries": 11, "ndcg@10": 0.6815199154995802, "recall@10": 0.4656565656565656, "mrr": 0.8484848484848485, "p50_retrieve_ms": 74.312, "p95_retrieve_ms": 91.006, "git_sha": "f42c4a8eaadb3c0cc79f78c34b0e1a70534bee10", "report_path": "runs/search-esci-slice-pe/report.json", "recorded_at": "2026-08-18T13:31:46.700938+00:00" }, { "run_id": "search-esci-slice-entities-after", "brain": "searchbenchesci20", "dataset": "search_esci_slice.jsonl", "fusion": "rrf", "rerank": "none", "channels": [ "entities" ], "n_queries": 11, "ndcg@10": 0.6549492067418874, "recall@10": 0.4818181818181818, "mrr": 0.7568181818181817, "p50_retrieve_ms": 51.071, "p95_retrieve_ms": 74.023, "git_sha": "f42c4a8eaadb3c0cc79f78c34b0e1a70534bee10", "report_path": "runs/search-esci-slice-entities-after/report.json", "recorded_at": "2026-08-18T13:36:54.694299+00:00" }, { "run_id": "search-esci-slice-communities-after", "brain": "searchbenchesci20", "dataset": "search_esci_slice.jsonl", "fusion": "rrf", "rerank": "none", "channels": [ "communities" ], "n_queries": 11, "ndcg@10": 0.4776363794553365, "recall@10": 0.35000000000000003, "mrr": 0.8484848484848485, "p50_retrieve_ms": 129.202, "p95_retrieve_ms": 152.889, "git_sha": "f42c4a8eaadb3c0cc79f78c34b0e1a70534bee10", "report_path": "runs/search-esci-slice-communities-after/report.json", "recorded_at": "2026-08-18T13:39:09.162227+00:00" }, { "run_id": "search-esci-slice-passages-smoke", "brain": "searchbenchesci20", "dataset": "search_esci_slice.jsonl", "fusion": "rrf", "rerank": "none", "channels": [ "passages" ], "n_queries": 11, "ndcg@10": 0.7579469533596556, "recall@10": 0.5106060606060606, "mrr": 0.8484848484848485, "p50_retrieve_ms": 27.735, "p95_retrieve_ms": 32.483000000000004, "git_sha": "f42c4a8eaadb3c0cc79f78c34b0e1a70534bee10", "report_path": "runs/search-esci-slice-passages-smoke/report.json", "recorded_at": "2026-08-18T13:37:03.038349+00:00" }, { "run_id": "search-esci-slice-pec-after", "brain": "searchbenchesci20", "dataset": "search_esci_slice.jsonl", "fusion": "rrf", "rerank": "none", "channels": [ "passages", "entities", "communities" ], "n_queries": 11, "ndcg@10": 0.6526549564168582, "recall@10": 0.496969696969697, "mrr": 0.816017316017316, "p50_retrieve_ms": 183.946, "p95_retrieve_ms": 216.89249999999998, "git_sha": "f42c4a8eaadb3c0cc79f78c34b0e1a70534bee10", "report_path": "runs/search-esci-slice-pec-after/report.json", "recorded_at": "2026-08-18T13:39:52.408131+00:00" }, { "run_id": "search-esci-slice-passages-ce", "brain": "searchbenchesci20", "dataset": "search_esci_slice.jsonl", "fusion": "rrf", "rerank": "plugin:cross-encoder", "channels": [ "passages" ], "rank_pool": false, "rank_pool_ce": false, "n_queries": 11, "ndcg@10": 0.7375861975343849, "ndcg@20": null, "ndcg": null, "recall@10": 0.5106060606060606, "recall@20": 0.8469696969696969, "mrr": 0.8030303030303031, "p50_retrieve_ms": 30.029, "p95_retrieve_ms": 76.32499999999999, "git_sha": "f42c4a8eaadb3c0cc79f78c34b0e1a70534bee10", "report_path": "runs/search-esci-slice-passages-ce/report.json", "recorded_at": "2026-08-18T14:20:28.184907+00:00" }, { "run_id": "search-esci-slice-passages-pool", "brain": "searchbenchesci20", "dataset": "search_esci_slice.jsonl", "fusion": "rrf", "rerank": "none", "channels": [ "passages" ], "rank_pool": true, "n_queries": 11, "ndcg@10": 0.7884517684883318, "ndcg@20": 0.8114424261982733, "ndcg": 0.8137804024940007, "recall@10": 0.5348484848484848, "recall@20": 0.8469696969696969, "mrr": 0.9090909090909091, "p50_retrieve_ms": 22.721, "p95_retrieve_ms": 75.3825, "git_sha": "f42c4a8eaadb3c0cc79f78c34b0e1a70534bee10", "report_path": "runs/search-esci-slice-passages-pool/report.json", "recorded_at": "2026-08-18T14:18:44.171034+00:00" }, { "run_id": "search-esci-slice-ce-pool", "brain": "searchbenchesci20", "dataset": "search_esci_slice.jsonl", "fusion": "none", "rerank": "plugin:cross-encoder", "channels": [ "rank-pool-ce" ], "rank_pool": true, "rank_pool_ce": true, "n_queries": 11, "ndcg@10": 0.5976277406374534, "ndcg@20": 0.668518640484444, "ndcg": 0.7862048091602294, "recall@10": 0.398637349324982, "recall@20": 0.6598423817505625, "mrr": 0.8333333333333333, "p50_retrieve_ms": 0.0, "p95_retrieve_ms": 0.0, "git_sha": "f42c4a8eaadb3c0cc79f78c34b0e1a70534bee10", "report_path": "runs/search-esci-slice-ce-pool/report.json", "recorded_at": "2026-08-18T14:20:28.034427+00:00" }, { "run_id": "search-esci-74-ce-pool", "brain": "searchbenchesci74", "dataset": "search_esci_74.jsonl", "fusion": "none", "rerank": "plugin:cross-encoder", "channels": [ "rank-pool-ce" ], "rank_pool": true, "rank_pool_ce": true, "n_queries": 74, "ndcg@10": 0.5492314171926193, "ndcg@20": 0.6311374744057928, "ndcg": 0.7174501167408996, "recall@10": 0.4907400145298616, "recall@20": 0.7577102510384188, "mrr": 0.8306570306570307, "p50_retrieve_ms": 0.0, "p95_retrieve_ms": 0.0, "git_sha": "f42c4a8eaadb3c0cc79f78c34b0e1a70534bee10", "report_path": "runs/search-esci-74-ce-pool/report.json", "recorded_at": "2026-08-18T14:29:17.114211+00:00" }, { "run_id": "search-esci-74-ce-pool-ft", "brain": "searchbenchesci74", "dataset": "search_esci_74.jsonl", "fusion": "none", "rerank": "plugin:cross-encoder", "channels": [ "rank-pool-ce" ], "rank_pool": true, "rank_pool_ce": true, "n_queries": 74, "ndcg@10": 0.5832632846066277, "ndcg@20": 0.6690889170924552, "ndcg": 0.7451408010463653, "recall@10": 0.4893954738298456, "recall@20": 0.7641337775411228, "mrr": 0.8932519057519057, "p50_retrieve_ms": 0.0, "p95_retrieve_ms": 0.0, "git_sha": "f42c4a8eaadb3c0cc79f78c34b0e1a70534bee10", "report_path": "runs/search-esci-74-ce-pool-ft/report.json", "recorded_at": "2026-08-18T14:32:48.672213+00:00" }, { "run_id": "search-esci-74-passages", "brain": "searchbenchesci74", "dataset": "search_esci_74.jsonl", "fusion": "rrf", "rerank": "none", "channels": [ "passages" ], "rank_pool": false, "rank_pool_ce": false, "n_queries": 74, "ndcg@10": 0.49528253069041633, "ndcg@20": 0.5490094202707876, "ndcg": 0.5490094202707876, "recall@10": 0.3769307112873572, "recall@20": 0.5920965629425753, "mrr": 0.7580598455598455, "p50_retrieve_ms": 70.9225, "p95_retrieve_ms": 94.17495, "git_sha": "f42c4a8eaadb3c0cc79f78c34b0e1a70534bee10", "report_path": "runs/search-esci-74-passages/report.json", "recorded_at": "2026-08-18T15:38:50.867049+00:00" }, { "run_id": "search-esci-74-passages-pool", "brain": "searchbenchesci74", "dataset": "search_esci_74.jsonl", "fusion": "rrf", "rerank": "none", "channels": [ "passages" ], "rank_pool": true, "rank_pool_ce": false, "n_queries": 74, "ndcg@10": 0.5458756376131827, "ndcg@20": 0.5785048208487761, "ndcg": 0.6013933046523523, "recall@10": 0.42064799804369324, "recall@20": 0.5920965629425753, "mrr": 0.8254504504504505, "p50_retrieve_ms": 65.447, "p95_retrieve_ms": 91.44999999999999, "git_sha": "f42c4a8eaadb3c0cc79f78c34b0e1a70534bee10", "report_path": "runs/search-esci-74-passages-pool/report.json", "recorded_at": "2026-08-18T15:41:37.970033+00:00" }, { "run_id": "search-esci-slice-passages-gate", "brain": "searchbenchesci20", "dataset": "search_esci_slice.jsonl", "fusion": "rrf", "rerank": "none", "channels": [ "passages" ], "rank_pool": false, "rank_pool_ce": false, "n_queries": 11, "ndcg@10": 0.7579469533596556, "ndcg@20": 0.7962544743266965, "ndcg": 0.7962544743266965, "recall@10": 0.5106060606060606, "recall@20": 0.8469696969696969, "mrr": 0.8484848484848485, "p50_retrieve_ms": 25.549, "p95_retrieve_ms": 41.532, "git_sha": "f42c4a8eaadb3c0cc79f78c34b0e1a70534bee10", "report_path": "runs/search-esci-slice-passages-gate/report.json", "recorded_at": "2026-08-18T16:47:11.911155+00:00" }, { "run_id": "search-esci-slice-pec-lists", "brain": "searchbenchesci20", "dataset": "search_esci_slice.jsonl", "fusion": "rrf", "rerank": "none", "channels": [ "passages", "entities", "communities" ], "rank_pool": false, "rank_pool_ce": false, "n_queries": 11, "ndcg@10": 0.6526549564168582, "ndcg@20": 0.6261389422887412, "ndcg": 0.683063799169122, "recall@10": 0.496969696969697, "recall@20": 0.6530303030303031, "mrr": 0.816017316017316, "p50_retrieve_ms": 160.231, "p95_retrieve_ms": 198.8395, "git_sha": "f42c4a8eaadb3c0cc79f78c34b0e1a70534bee10", "report_path": "runs/search-esci-slice-pec-lists/report.json", "recorded_at": "2026-08-18T16:47:26.669145+00:00" }, { "run_id": "search-esci-slice-entities-neighbors", "brain": "searchbenchesci20", "dataset": "search_esci_slice.jsonl", "fusion": "rrf", "rerank": "none", "channels": [ "entities" ], "rank_pool": false, "rank_pool_ce": false, "n_queries": 11, "ndcg@10": 0.6549492067418874, "ndcg@20": 0.7173885785786568, "ndcg": 0.7173885785786568, "recall@10": 0.4818181818181818, "recall@20": 0.8217171717171716, "mrr": 0.7568181818181817, "p50_retrieve_ms": 223.289, "p95_retrieve_ms": 271.031, "git_sha": "f42c4a8eaadb3c0cc79f78c34b0e1a70534bee10", "report_path": "runs/search-esci-slice-entities-neighbors/report.json", "recorded_at": "2026-08-18T16:48:04.631525+00:00" }, { "run_id": "search-esci-74-passages-control", "brain": "searchbenchesci74", "dataset": "search_esci_74.jsonl", "fusion": "rrf", "rerank": "none", "channels": [ "passages" ], "rank_pool": false, "rank_pool_ce": false, "n_queries": 74, "ndcg@10": 0.49528253069041633, "ndcg@20": 0.5490094202707876, "ndcg": 0.5490094202707876, "recall@10": 0.3769307112873572, "recall@20": 0.5920965629425753, "mrr": 0.7580598455598455, "p50_retrieve_ms": 61.7115, "p95_retrieve_ms": 91.52104999999995, "git_sha": "f42c4a8eaadb3c0cc79f78c34b0e1a70534bee10", "report_path": "runs/search-esci-74-passages-control/report.json", "recorded_at": "2026-08-18T17:13:00.463193+00:00" }, { "run_id": "search-esci-74-ce-pool-l12-4class", "brain": "searchbenchesci74", "dataset": "search_esci_74.jsonl", "fusion": "none", "rerank": "4class-weighted-ce", "channels": [ "rank-pool-4class" ], "rank_pool": true, "rank_pool_ce": true, "n_queries": 74, "ndcg@10": 0.5968432958924662, "ndcg@20": 0.6782580445181766, "ndcg": 0.7467109659912253, "recall@10": 0.5088965287461584, "recall@20": 0.7673275730894211, "mrr": 0.8351464398761695, "p50_retrieve_ms": 0.0, "p95_retrieve_ms": 0.0, "git_sha": "f42c4a8eaadb3c0cc79f78c34b0e1a70534bee10", "report_path": "runs/search-esci-74-ce-pool-l12-4class/report.json", "recorded_at": "2026-08-18T17:44:17.593025+00:00" }, { "run_id": "search-esci-74-ce-pool-4class", "brain": "searchbenchesci74", "dataset": "search_esci_74.jsonl", "fusion": "none", "rerank": "4class-weighted-ce", "channels": [ "rank-pool-4class" ], "rank_pool": true, "rank_pool_ce": true, "n_queries": 74, "ndcg@10": 0.5826375707951627, "ndcg@20": 0.6696127031036031, "ndcg": 0.7419804620261266, "recall@10": 0.49798997460620914, "recall@20": 0.7668403956654921, "mrr": 0.8317567567567568, "p50_retrieve_ms": 0.0, "p95_retrieve_ms": 0.0, "git_sha": "f42c4a8eaadb3c0cc79f78c34b0e1a70534bee10", "report_path": "runs/search-esci-74-ce-pool-4class/report.json", "recorded_at": "2026-08-18T17:54:14.073711+00:00" }, { "run_id": "search-esci-74-ce-pool-4class-nowt", "brain": "searchbenchesci74", "dataset": "search_esci_74.jsonl", "fusion": "none", "rerank": "4class-weighted-ce", "channels": [ "rank-pool-4class" ], "rank_pool": true, "rank_pool_ce": true, "n_queries": 74, "ndcg@10": 0.6096378704103984, "ndcg@20": 0.6881968860231006, "ndcg": 0.7594213054854562, "recall@10": 0.5249760000665886, "recall@20": 0.7631438693842529, "mrr": 0.8530036338859869, "p50_retrieve_ms": 0.0, "p95_retrieve_ms": 0.0, "git_sha": "f42c4a8eaadb3c0cc79f78c34b0e1a70534bee10", "report_path": "runs/search-esci-74-ce-pool-4class-nowt/report.json", "recorded_at": "2026-08-18T18:20:20.750483+00:00" }, { "run_id": "search-esci-74-ce-pool-4class-e2", "brain": "searchbenchesci74", "dataset": "search_esci_74.jsonl", "fusion": "none", "rerank": "4class-weighted-ce", "channels": [ "rank-pool-4class" ], "rank_pool": true, "rank_pool_ce": true, "n_queries": 74, "ndcg@10": 0.6169746541917066, "ndcg@20": 0.6945278030601721, "ndcg": 0.7652624576380807, "recall@10": 0.5214445444747297, "recall@20": 0.7719306369596903, "mrr": 0.8525418275418275, "p50_retrieve_ms": 0.0, "p95_retrieve_ms": 0.0, "git_sha": "f42c4a8eaadb3c0cc79f78c34b0e1a70534bee10", "report_path": "runs/search-esci-74-ce-pool-4class-e2/report.json", "recorded_at": "2026-08-18T18:48:00.939749+00:00" }, { "run_id": "search-esci-74-passages-k50", "brain": "searchbenchesci74", "dataset": "search_esci_74.jsonl", "fusion": "rrf", "rerank": "none", "channels": [ "passages" ], "rank_pool": false, "rank_pool_ce": false, "n_queries": 74, "k": 50, "ndcg@10": 0.4998984443226263, "ndcg@20": 0.5522842574567189, "ndcg@50": 0.6378451547901237, "ndcg": 0.6378451547901237, "recall@10": 0.378732513089159, "recall@20": 0.589865760711773, "recall@50": 0.833954370548067, "mrr": 0.7645849420849421, "pool_coverage": null, "p50_retrieve_ms": 60.091, "p95_retrieve_ms": 88.02255, "git_sha": "f42c4a8eaadb3c0cc79f78c34b0e1a70534bee10", "report_path": "runs/search-esci-74-passages-k50/report.json", "recorded_at": "2026-08-18T20:31:55.998500+00:00" }, { "run_id": "search-esci-74-passages-k100", "brain": "searchbenchesci74", "dataset": "search_esci_74.jsonl", "fusion": "rrf", "rerank": "none", "channels": [ "passages" ], "rank_pool": false, "rank_pool_ce": false, "n_queries": 74, "k": 100, "ndcg@10": 0.4998055248060819, "ndcg@20": 0.5523315287868084, "ndcg@50": 0.6375525242924566, "ndcg": 0.6564185982755066, "recall@10": 0.378732513089159, "recall@20": 0.5913672622132746, "recall@50": 0.830951367545064, "mrr": 0.76497727997728, "pool_coverage": null, "p50_retrieve_ms": 57.553, "p95_retrieve_ms": 81.81569999999999, "git_sha": "f42c4a8eaadb3c0cc79f78c34b0e1a70534bee10", "report_path": "runs/search-esci-74-passages-k100/report.json", "recorded_at": "2026-08-18T20:32:27.553496+00:00" }, { "run_id": "search-esci-74-passages-k50-pool", "brain": "searchbenchesci74", "dataset": "search_esci_74.jsonl", "fusion": "rrf", "rerank": "none", "channels": [ "passages" ], "rank_pool": true, "rank_pool_ce": false, "n_queries": 74, "k": 50, "ndcg@10": 0.5668024300832123, "ndcg@20": 0.624102711461205, "ndcg@50": 0.6826188564511254, "ndcg": 0.6947261855882534, "recall@10": 0.4507852781809733, "recall@20": 0.6686543312372134, "recall@50": 0.8324528690465655, "mrr": 0.8271396396396397, "pool_coverage": 0.778222315718655, "p50_retrieve_ms": 60.1315, "p95_retrieve_ms": 86.49954999999997, "git_sha": "f42c4a8eaadb3c0cc79f78c34b0e1a70534bee10", "report_path": "runs/search-esci-74-passages-k50-pool/report.json", "recorded_at": "2026-08-18T20:33:02.203247+00:00" }, { "run_id": "search-esci-74-passages-k100-pool", "brain": "searchbenchesci74", "dataset": "search_esci_74.jsonl", "fusion": "rrf", "rerank": "none", "channels": [ "passages" ], "rank_pool": true, "rank_pool_ce": false, "n_queries": 74, "k": 100, "ndcg@10": 0.5737527941225129, "ndcg@20": 0.6352462971440838, "ndcg@50": 0.7092403328584962, "ndcg": 0.7183777959320791, "recall@10": 0.46580029319598837, "recall@20": 0.6964864396614271, "recall@50": 0.9062083174986402, "mrr": 0.836599099099099, "pool_coverage": 0.8462618325454071, "p50_retrieve_ms": 58.5245, "p95_retrieve_ms": 82.5653, "git_sha": "f42c4a8eaadb3c0cc79f78c34b0e1a70534bee10", "report_path": "runs/search-esci-74-passages-k100-pool/report.json", "recorded_at": "2026-08-18T20:33:39.620042+00:00" }, { "run_id": "search-esci-74-passages-k50-ce", "brain": "searchbenchesci74", "dataset": "search_esci_74.jsonl", "fusion": "rrf", "rerank": "harness:cross-encoder", "channels": [ "passages" ], "rank_pool": false, "rank_pool_ce": false, "n_queries": 74, "k": 50, "ndcg@10": 0.4478964444706324, "ndcg@20": 0.5146313751051845, "ndcg@50": 0.6060742524325241, "ndcg": 0.6060742524325241, "recall@10": 0.331273452580599, "recall@20": 0.5538599992903688, "recall@50": 0.833954370548067, "mrr": 0.7197929141477528, "pool_coverage": null, "p50_retrieve_ms": 60.091, "p95_retrieve_ms": 88.02255, "git_sha": "f42c4a8eaadb3c0cc79f78c34b0e1a70534bee10", "report_path": "runs/search-esci-74-passages-k50-ce/report.json", "recorded_at": "2026-08-18T20:34:39.854467+00:00" }, { "run_id": "search-esci-74-passages-k50-ce-l12", "brain": "searchbenchesci74", "dataset": "search_esci_74.jsonl", "fusion": "rrf", "rerank": "harness:cross-encoder", "channels": [ "passages" ], "rank_pool": false, "rank_pool_ce": false, "n_queries": 74, "k": 50, "ndcg@10": 0.4674585131694284, "ndcg@20": 0.5084668214514952, "ndcg@50": 0.6140738833603318, "ndcg": 0.6140738833603318, "recall@10": 0.3412172585548375, "recall@20": 0.5191872363321448, "recall@50": 0.833954370548067, "mrr": 0.7063349063349063, "pool_coverage": null, "p50_retrieve_ms": 60.091, "p95_retrieve_ms": 88.02255, "git_sha": "f42c4a8eaadb3c0cc79f78c34b0e1a70534bee10", "report_path": "runs/search-esci-74-passages-k50-ce-l12/report.json", "recorded_at": "2026-08-18T20:35:44.462471+00:00" }, { "run_id": "search-esci-74-passages-k50-qrewrite", "brain": "searchbenchesci74", "dataset": "search_esci_74_qrewrite.jsonl", "fusion": "rrf", "rerank": "none", "channels": [ "passages" ], "rank_pool": false, "rank_pool_ce": false, "n_queries": 74, "k": 50, "ndcg@10": 0.4998984443226263, "ndcg@20": 0.5522842574567189, "ndcg@50": 0.6378451547901237, "ndcg": 0.6378451547901237, "recall@10": 0.378732513089159, "recall@20": 0.589865760711773, "recall@50": 0.833954370548067, "mrr": 0.7645849420849421, "pool_coverage": null, "p50_retrieve_ms": 59.432500000000005, "p95_retrieve_ms": 82.09889999999999, "git_sha": "f42c4a8eaadb3c0cc79f78c34b0e1a70534bee10", "report_path": "runs/search-esci-74-passages-k50-qrewrite/report.json", "recorded_at": "2026-08-18T21:18:51.308816+00:00" }, { "run_id": "search-esci-74-dense-ance-k50", "brain": "harness-local", "dataset": "search_esci_74.jsonl", "fusion": "none", "rerank": "none", "channels": [ "harness-dense" ], "rank_pool": false, "rank_pool_ce": false, "n_queries": 74, "k": 50, "ndcg@10": 0.3960504272587966, "ndcg@20": 0.445560860839057, "ndcg@50": 0.5370854731938006, "ndcg": 0.5370854731938006, "recall@10": 0.3015013697338196, "recall@20": 0.4872087291019431, "recall@50": 0.7503156876126188, "mrr": 0.6471417846417846, "pool_coverage": null, "p50_retrieve_ms": 74.9370945944778, "p95_retrieve_ms": 74.9370945944778, "git_sha": "f42c4a8eaadb3c0cc79f78c34b0e1a70534bee10", "report_path": "runs/search-esci-74-dense-ance-k50/report.json", "recorded_at": "2026-08-18T21:22:24.054560+00:00" }, { "run_id": "search-esci-74-passages-k50-ce-retrieved", "brain": "searchbenchesci74", "dataset": "search_esci_74.jsonl", "fusion": "rrf", "rerank": "harness:cross-encoder", "channels": [ "passages" ], "rank_pool": false, "rank_pool_ce": false, "n_queries": 74, "k": 50, "ndcg@10": 0.4648895994453083, "ndcg@20": 0.5245865140860362, "ndcg@50": 0.6126753191137789, "ndcg": 0.6126753191137789, "recall@10": 0.35140463046557896, "recall@20": 0.5410788442403298, "recall@50": 0.833954370548067, "mrr": 0.7120610588001892, "pool_coverage": null, "p50_retrieve_ms": 60.091, "p95_retrieve_ms": 88.02255, "git_sha": "f42c4a8eaadb3c0cc79f78c34b0e1a70534bee10", "report_path": "runs/search-esci-74-passages-k50-ce-retrieved/report.json", "recorded_at": "2026-08-18T22:38:31.141079+00:00" }, { "run_id": "search-esci-74-colbert-k50", "brain": "harness-local-colbert", "dataset": "search_esci_74.jsonl", "fusion": "none", "rerank": "none", "channels": [ "harness-colbert" ], "rank_pool": false, "rank_pool_ce": false, "n_queries": 74, "k": 50, "ndcg@10": 0.43412701768229023, "ndcg@20": 0.4815601734708754, "ndcg@50": 0.5523204205368764, "ndcg": 0.5523204205368764, "recall@10": 0.3110168068124024, "recall@20": 0.5023603484326227, "recall@50": 0.7138820979266567, "mrr": 0.6689285195333583, "pool_coverage": null, "p50_retrieve_ms": 10063.364791662614, "p95_retrieve_ms": 10063.364791662614, "git_sha": "f42c4a8eaadb3c0cc79f78c34b0e1a70534bee10", "report_path": "runs/search-esci-74-colbert-k50/report.json", "recorded_at": "2026-08-18T22:51:16.979167+00:00" }, { "run_id": "search-esci-74-bge-base-k50", "brain": "harness-local", "dataset": "search_esci_74.jsonl", "fusion": "none", "rerank": "none", "channels": [ "harness-dense" ], "rank_pool": false, "rank_pool_ce": false, "n_queries": 74, "k": 50, "ndcg@10": 0.44136708015333975, "ndcg@20": 0.5041842920107446, "ndcg@50": 0.5892299329783445, "ndcg": 0.5892299329783445, "recall@10": 0.32179328339461255, "recall@20": 0.5251414009467592, "recall@50": 0.7989247943561789, "mrr": 0.7471303646913403, "pool_coverage": null, "p50_retrieve_ms": 408.0482843509762, "p95_retrieve_ms": 408.04828435097625, "git_sha": "f42c4a8eaadb3c0cc79f78c34b0e1a70534bee10", "report_path": "runs/search-esci-74-bge-base-k50/report.json", "recorded_at": "2026-08-18T22:52:53.742680+00:00" }, { "run_id": "search-esci-74-union-bge-k50", "brain": "harness-local-union", "dataset": "search_esci_74.jsonl", "fusion": "rrf", "rerank": "none", "channels": [ "harness-union" ], "rank_pool": false, "rank_pool_ce": false, "n_queries": 74, "k": 50, "ndcg@10": 0.47918374395281466, "ndcg@20": 0.5381296158524957, "ndcg@50": 0.6141255447421892, "ndcg": 0.6141255447421892, "recall@10": 0.36314016383757297, "recall@20": 0.5774058852814317, "recall@50": 0.8271070974286027, "mrr": 0.7653163449499656, "pool_coverage": null, "p50_retrieve_ms": 0.0, "p95_retrieve_ms": 0.0, "git_sha": "f42c4a8eaadb3c0cc79f78c34b0e1a70534bee10", "report_path": "runs/search-esci-74-union-bge-k50/report.json", "recorded_at": "2026-08-19T08:10:06.928384+00:00" }, { "run_id": "search-esci-74-exhaustive-ce", "brain": "harness-local-exhaustive", "dataset": "search_esci_74.jsonl", "fusion": "none", "rerank": "4class-weighted-ce", "channels": [ "exhaustive-4class" ], "rank_pool": false, "rank_pool_ce": false, "n_queries": 74, "k": 50, "ndcg@10": 0.4162185585932262, "ndcg@20": 0.4392534244105597, "ndcg@50": 0.5042093175793891, "ndcg": 0.5042093175793891, "recall@10": 0.26585896457775815, "recall@20": 0.4060812399456508, "recall@50": 0.6029270278490679, "mrr": 0.6578079129936715, "pool_coverage": null, "p50_retrieve_ms": 94253.11211881155, "p95_retrieve_ms": 94253.11211881155, "git_sha": "f42c4a8eaadb3c0cc79f78c34b0e1a70534bee10", "report_path": "runs/search-esci-74-exhaustive-ce/report.json", "recorded_at": "2026-08-19T10:05:59.463503+00:00" }, { "run_id": "search-esci-74-catalog-ce-k50", "brain": "searchbenchesci74", "dataset": "search_esci_74.jsonl", "fusion": "rrf", "rerank": "plugin:cross-encoder", "channels": [ "passages" ], "rank_pool": false, "rank_pool_ce": false, "n_queries": 74, "k": 50, "ndcg@10": 0.4674585131694284, "ndcg@20": 0.5084668214514952, "ndcg@50": 0.6140738833603318, "ndcg": 0.6140738833603318, "recall@10": 0.3412172585548375, "recall@20": 0.5191872363321448, "recall@50": 0.833954370548067, "mrr": 0.7063349063349063, "pool_coverage": null, "p50_retrieve_ms": 59.308499999999995, "p95_retrieve_ms": 86.88884999999999, "git_sha": "f42c4a8eaadb3c0cc79f78c34b0e1a70534bee10", "report_path": "runs/search-esci-74-catalog-ce-k50/report.json", "recorded_at": "2026-08-19T10:17:45.568452+00:00" }, { "run_id": "search-esci-74-cascade-tail-k50", "brain": "harness-local-cascade", "dataset": "search_esci_74.jsonl", "fusion": "none", "rerank": "none", "channels": [ "harness-cascade" ], "rank_pool": false, "rank_pool_ce": false, "n_queries": 74, "k": 50, "ndcg@10": 0.4998984443226263, "ndcg@20": 0.6194622913116794, "ndcg@50": 0.6686100079902446, "ndcg": 0.6686100079902446, "recall@10": 0.378732513089159, "recall@20": 0.7504072036031975, "recall@50": 0.8885964311815054, "mrr": 0.7679931554931555, "pool_coverage": null, "p50_retrieve_ms": 0.0, "p95_retrieve_ms": 0.0, "git_sha": "f42c4a8eaadb3c0cc79f78c34b0e1a70534bee10", "report_path": "runs/search-esci-74-cascade-tail-k50/report.json", "recorded_at": "2026-08-19T10:32:17.738702+00:00" }, { "run_id": "search-esci-74-ce-pool-deberta-base", "brain": "searchbenchesci74", "dataset": "search_esci_74.jsonl", "fusion": "none", "rerank": "4class-weighted-ce", "channels": [ "rank-pool-4class" ], "rank_pool": true, "rank_pool_ce": true, "n_queries": 74, "k": 40, "ndcg@10": 0.6492728519674476, "ndcg@20": 0.7101117922903459, "ndcg@50": null, "ndcg": 0.7772540710194906, "recall@10": 0.5183145850778, "recall@20": 0.7604145345598889, "recall@50": null, "mrr": 0.8646712599298806, "pool_coverage": null, "p50_retrieve_ms": 0.0, "p95_retrieve_ms": 0.0, "git_sha": "f42c4a8eaadb3c0cc79f78c34b0e1a70534bee10", "report_path": "runs/search-esci-74-ce-pool-deberta-base/report.json", "recorded_at": "2026-08-19T14:00:25.594211+00:00" }, { "run_id": "search-esci-74-passages-k50-ce-deberta", "brain": "searchbenchsmoke", "dataset": "search_esci_74.jsonl", "fusion": "rrf", "rerank": "harness:cross-encoder", "channels": [ "passages" ], "rank_pool": false, "rank_pool_ce": false, "n_queries": 74, "k": 50, "ndcg@10": 0.5101053946025641, "ndcg@20": 0.5568741362838021, "ndcg@50": 0.6365973490191127, "ndcg": 0.6365973490191127, "recall@10": 0.36281504203381093, "recall@20": 0.5467544580912761, "recall@50": 0.833954370548067, "mrr": 0.7457139784725992, "pool_coverage": null, "p50_retrieve_ms": 60.091, "p95_retrieve_ms": 88.02255, "git_sha": "f42c4a8eaadb3c0cc79f78c34b0e1a70534bee10", "report_path": "runs/search-esci-74-passages-k50-ce-deberta/report.json", "recorded_at": "2026-08-19T14:09:19.540811+00:00" }, { "run_id": "search-esci-es-passages-k50", "brain": "searchbenchescies", "dataset": "search_esci_es.jsonl", "fusion": "rrf", "rerank": "none", "channels": [ "passages" ], "rank_pool": false, "rank_pool_ce": false, "n_queries": 62, "k": 50, "ndcg@10": 0.5774393479403104, "ndcg@20": 0.6303553323648456, "ndcg@50": 0.7327853549091549, "ndcg": 0.7327853549091549, "recall@10": 0.35328224222955834, "recall@20": 0.5758587932370467, "recall@50": 0.9141280526823616, "mrr": 0.8704493087557603, "pool_coverage": null, "p50_retrieve_ms": 84.13, "p95_retrieve_ms": 96.16295000000001, "git_sha": "f42c4a8eaadb3c0cc79f78c34b0e1a70534bee10", "report_path": "runs/search-esci-es-passages-k50/report.json", "recorded_at": "2026-08-19T16:34:42.965337+00:00" }, { "run_id": "search-italian-smoke", "brain": "searchbenchitsmoke", "dataset": "search_italian_smoke.jsonl", "fusion": "rrf", "rerank": "none", "channels": [ "passages" ], "rank_pool": false, "rank_pool_ce": false, "n_queries": 3, "k": 20, "ndcg@10": 1.0, "ndcg@20": 1.0, "ndcg@50": null, "ndcg": 1.0, "recall@10": 1.0, "recall@20": 1.0, "recall@50": null, "mrr": 1.0, "pool_coverage": null, "p50_retrieve_ms": 25.33, "p95_retrieve_ms": 29.3206, "git_sha": "f42c4a8eaadb3c0cc79f78c34b0e1a70534bee10", "report_path": "runs/search-italian-smoke/report.json", "recorded_at": "2026-08-19T16:37:37.730780+00:00" }, { "run_id": "search-italian-smoke-inflect", "brain": "searchbenchitsmoke", "dataset": "search_italian_smoke_inflect.jsonl", "fusion": "rrf", "rerank": "none", "channels": [ "passages" ], "rank_pool": false, "rank_pool_ce": false, "n_queries": 3, "k": 20, "ndcg@10": 1.0, "ndcg@20": 1.0, "ndcg@50": null, "ndcg": 1.0, "recall@10": 1.0, "recall@20": 1.0, "recall@50": null, "mrr": 1.0, "pool_coverage": null, "p50_retrieve_ms": 21.494, "p95_retrieve_ms": 23.211199999999998, "git_sha": "f42c4a8eaadb3c0cc79f78c34b0e1a70534bee10", "report_path": "runs/search-italian-smoke-inflect/report.json", "recorded_at": "2026-08-19T16:37:46.108861+00:00" }, { "run_id": "search-italian-smoke-inflect-fts", "brain": "searchbenchitsmoke", "dataset": "search_italian_smoke_inflect.jsonl", "fusion": "rrf", "rerank": "none", "channels": [ "passages" ], "rank_pool": false, "rank_pool_ce": false, "n_queries": 3, "k": 20, "ndcg@10": 1.0, "ndcg@20": 1.0, "ndcg@50": null, "ndcg": 1.0, "recall@10": 1.0, "recall@20": 1.0, "recall@50": null, "mrr": 1.0, "pool_coverage": null, "p50_retrieve_ms": 33.56, "p95_retrieve_ms": 36.8936, "git_sha": "f42c4a8eaadb3c0cc79f78c34b0e1a70534bee10", "report_path": "runs/search-italian-smoke-inflect-fts/report.json", "recorded_at": "2026-08-19T16:44:05.699330+00:00" }, { "run_id": "search-italian-minilm-pipeline", "brain": "searchbenchitmmini", "dataset": "search_italian_smoke.jsonl", "fusion": "none", "rerank": "none", "channels": [ "harness-dense" ], "rank_pool": false, "rank_pool_ce": false, "n_queries": 3, "k": 10, "ndcg@10": 1.0, "ndcg@20": 1.0, "ndcg@50": null, "ndcg": 1.0, "recall@10": 1.0, "recall@20": null, "recall@50": null, "mrr": 1.0, "pool_coverage": null, "p50_retrieve_ms": 188.19916666689096, "p95_retrieve_ms": 188.19916666689096, "git_sha": "f42c4a8eaadb3c0cc79f78c34b0e1a70534bee10", "report_path": "runs/search-italian-minilm-pipeline/report.json", "recorded_at": "2026-08-19T19:27:33.792776+00:00" }, { "run_id": "search-esci-es-spell-k50", "brain": "searchbenchescies", "dataset": "search_esci_es_spell.jsonl", "fusion": "rrf", "rerank": "none", "channels": [ "passages" ], "rank_pool": false, "rank_pool_ce": false, "n_queries": 62, "k": 50, "ndcg@10": 0.5950327313726855, "ndcg@20": 0.6464666665333806, "ndcg@50": 0.7424579237775919, "ndcg": 0.7424579237775919, "recall@10": 0.36220476611566177, "recall@20": 0.5832920760052671, "recall@50": 0.9143064382925891, "mrr": 0.8865783410138248, "pool_coverage": null, "p50_retrieve_ms": 92.655, "p95_retrieve_ms": 107.2923, "git_sha": "f42c4a8eaadb3c0cc79f78c34b0e1a70534bee10", "report_path": "runs/search-esci-es-spell-k50/report.json", "recorded_at": "2026-08-19T19:28:14.068481+00:00" }, { "run_id": "search-italian-smoke-extras-locale", "brain": "searchbenchitsmoke", "dataset": "search_italian_smoke.jsonl", "fusion": "rrf", "rerank": "none", "channels": [ "passages" ], "rank_pool": false, "rank_pool_ce": false, "n_queries": 3, "k": 10, "ndcg@10": 1.0, "ndcg@20": 1.0, "ndcg@50": null, "ndcg": 1.0, "recall@10": 1.0, "recall@20": null, "recall@50": null, "mrr": 1.0, "pool_coverage": null, "p50_retrieve_ms": 17.126, "p95_retrieve_ms": 23.894000000000002, "git_sha": "f42c4a8eaadb3c0cc79f78c34b0e1a70534bee10", "report_path": "runs/search-italian-smoke-extras-locale/report.json", "recorded_at": "2026-08-19T19:30:47.530480+00:00" }, { "run_id": "search-italian-smoke-extras-nope", "brain": "searchbenchitsmoke", "dataset": "search_italian_smoke.jsonl", "fusion": "rrf", "rerank": "none", "channels": [ "passages" ], "rank_pool": false, "rank_pool_ce": false, "n_queries": 3, "k": 10, "ndcg@10": 0.0, "ndcg@20": 0.0, "ndcg@50": null, "ndcg": 0.0, "recall@10": 0.0, "recall@20": null, "recall@50": null, "mrr": 0.0, "pool_coverage": null, "p50_retrieve_ms": 22.721, "p95_retrieve_ms": 24.2753, "git_sha": "f42c4a8eaadb3c0cc79f78c34b0e1a70534bee10", "report_path": "runs/search-italian-smoke-extras-nope/report.json", "recorded_at": "2026-08-19T19:29:31.676418+00:00" }, { "run_id": "search-esci-74-passages-k50-extras-sanity", "brain": "searchbenchesci74", "dataset": "search_esci_74.jsonl", "fusion": "rrf", "rerank": "none", "channels": [ "passages" ], "rank_pool": false, "rank_pool_ce": false, "n_queries": 74, "k": 50, "ndcg@10": 0.4998984443226263, "ndcg@20": 0.5522842574567189, "ndcg@50": 0.6378451547901237, "ndcg": 0.6378451547901237, "recall@10": 0.378732513089159, "recall@20": 0.589865760711773, "recall@50": 0.833954370548067, "mrr": 0.7645849420849421, "pool_coverage": null, "p50_retrieve_ms": 63.1325, "p95_retrieve_ms": 88.86579999999998, "git_sha": "f42c4a8eaadb3c0cc79f78c34b0e1a70534bee10", "report_path": "runs/search-esci-74-passages-k50-extras-sanity/report.json", "recorded_at": "2026-08-19T19:35:44.545077+00:00" }, { "run_id": "search-italian-smoke-inflect-fts-or", "brain": "searchbenchitsmoke", "dataset": "search_italian_smoke_inflect.jsonl", "fusion": "rrf", "rerank": "none", "channels": [ "passages" ], "rank_pool": false, "rank_pool_ce": false, "n_queries": 3, "k": 20, "ndcg@10": 1.0, "ndcg@20": 1.0, "ndcg@50": null, "ndcg": 1.0, "recall@10": 1.0, "recall@20": 1.0, "recall@50": null, "mrr": 1.0, "pool_coverage": null, "p50_retrieve_ms": 26.364, "p95_retrieve_ms": 42.744899999999994, "git_sha": "f42c4a8eaadb3c0cc79f78c34b0e1a70534bee10", "report_path": "runs/search-italian-smoke-inflect-fts-or/report.json", "recorded_at": "2026-08-19T20:16:24.241357+00:00" }, { "run_id": "search-esci-74-cc-a03", "brain": "searchbenchesci74", "dataset": "search_esci_74.jsonl", "fusion": "cc", "rerank": "none", "channels": [ "passages" ], "rank_pool": false, "rank_pool_ce": false, "n_queries": 74, "k": 50, "ndcg@10": 0.48296913346675313, "ndcg@20": 0.5353434819101804, "ndcg@50": 0.6235227747628354, "ndcg": 0.6235227747628354, "recall@10": 0.37594464118364007, "recall@20": 0.5859902261910772, "recall@50": 0.8324528690465655, "mrr": 0.7492780202562811, "pool_coverage": null, "p50_retrieve_ms": 62.8885, "p95_retrieve_ms": 89.98204999999999, "git_sha": "f42c4a8eaadb3c0cc79f78c34b0e1a70534bee10", "report_path": "runs/search-esci-74-cc-a03/report.json", "recorded_at": "2026-08-19T20:18:18.361436+00:00" }, { "run_id": "search-esci-74-cc-a05", "brain": "searchbenchesci74", "dataset": "search_esci_74.jsonl", "fusion": "cc", "rerank": "none", "channels": [ "passages" ], "rank_pool": false, "rank_pool_ce": false, "n_queries": 74, "k": 50, "ndcg@10": 0.4861505982058926, "ndcg@20": 0.5412686017716648, "ndcg@50": 0.62608241624179, "ndcg": 0.62608241624179, "recall@10": 0.37673766109430706, "recall@20": 0.5919104321112831, "recall@50": 0.8324528690465655, "mrr": 0.7604021879021879, "pool_coverage": null, "p50_retrieve_ms": 61.432, "p95_retrieve_ms": 92.56864999999999, "git_sha": "f42c4a8eaadb3c0cc79f78c34b0e1a70534bee10", "report_path": "runs/search-esci-74-cc-a05/report.json", "recorded_at": "2026-08-19T20:18:41.196658+00:00" }, { "run_id": "search-esci-74-cc-a07", "brain": "searchbenchesci74", "dataset": "search_esci_74.jsonl", "fusion": "cc", "rerank": "none", "channels": [ "passages" ], "rank_pool": false, "rank_pool_ce": false, "n_queries": 74, "k": 50, "ndcg@10": 0.4925393217389386, "ndcg@20": 0.5491258060245199, "ndcg@50": 0.6321844176410597, "ndcg": 0.6321844176410597, "recall@10": 0.3780890124456584, "recall@20": 0.5964401922862047, "recall@50": 0.833954370548067, "mrr": 0.7630083655083656, "pool_coverage": null, "p50_retrieve_ms": 60.5155, "p95_retrieve_ms": 89.04719999999998, "git_sha": "f42c4a8eaadb3c0cc79f78c34b0e1a70534bee10", "report_path": "runs/search-esci-74-cc-a07/report.json", "recorded_at": "2026-08-19T20:19:05.271923+00:00" }, { "run_id": "search-esci-74-literal-fill-k50", "brain": "searchbenchesci74", "dataset": "search_esci_74.jsonl", "fusion": "rrf", "rerank": "none", "channels": [ "passages" ], "rank_pool": false, "rank_pool_ce": false, "n_queries": 74, "k": 50, "ndcg@10": 0.4998984443226263, "ndcg@20": 0.45027838324634667, "ndcg@50": 0.5436008239474691, "ndcg": 0.5436008239474691, "recall@10": 0.378732513089159, "recall@20": 0.3929861145019264, "recall@50": 0.6549969089908488, "mrr": 0.7639237451737452, "pool_coverage": null, "p50_retrieve_ms": 49.019999999999996, "p95_retrieve_ms": 74.27314999999999, "git_sha": "f42c4a8eaadb3c0cc79f78c34b0e1a70534bee10", "report_path": "runs/search-esci-74-literal-fill-k50/report.json", "recorded_at": "2026-08-19T20:20:15.131585+00:00" }, { "run_id": "search-esci-74-ltr-head-k50", "brain": "harness-local-ltr", "dataset": "search_esci_74.jsonl", "fusion": "none", "rerank": "ltr-pairwise", "channels": [ "harness-ltr-head" ], "rank_pool": false, "rank_pool_ce": false, "n_queries": 74, "k": 50, "ndcg@10": 0.5146015046757452, "ndcg@20": 0.5567887112159354, "ndcg@50": 0.6406685515140392, "ndcg": 0.6406685515140392, "recall@10": 0.384165323643092, "recall@20": 0.5830174861607277, "recall@50": 0.833954370548067, "mrr": 0.7525579150579151, "pool_coverage": null, "p50_retrieve_ms": 0.0, "p95_retrieve_ms": 0.0, "git_sha": "f42c4a8eaadb3c0cc79f78c34b0e1a70534bee10", "report_path": "runs/search-esci-74-ltr-head-k50/report.json", "recorded_at": "2026-08-19T21:09:47.411344+00:00" }, { "run_id": "search-esci-74-ltr-cefeat-k50", "brain": "harness-local-ltr", "dataset": "search_esci_74.jsonl", "fusion": "none", "rerank": "ltr-pairwise", "channels": [ "harness-ltr-head" ], "rank_pool": false, "rank_pool_ce": false, "n_queries": 74, "k": 50, "ndcg@10": 0.5242780297400812, "ndcg@20": 0.5655195290162985, "ndcg@50": 0.6456032400547804, "ndcg": 0.6456032400547804, "recall@10": 0.38343544674019964, "recall@20": 0.5687827599719494, "recall@50": 0.833954370548067, "mrr": 0.7792567567567568, "pool_coverage": null, "p50_retrieve_ms": 0.0, "p95_retrieve_ms": 0.0, "git_sha": "f42c4a8eaadb3c0cc79f78c34b0e1a70534bee10", "report_path": "runs/search-esci-74-ltr-cefeat-k50/report.json", "recorded_at": "2026-08-20T07:14:39.761450+00:00" }, { "run_id": "search-esci-74-ltr-deberta-k50", "brain": "harness-local-ltr", "dataset": "search_esci_74.jsonl", "fusion": "none", "rerank": "ltr-pairwise", "channels": [ "harness-ltr-head" ], "rank_pool": false, "rank_pool_ce": false, "n_queries": 74, "k": 50, "ndcg@10": 0.5420964444237363, "ndcg@20": 0.580013195834899, "ndcg@50": 0.6509103805030941, "ndcg": 0.6509103805030941, "recall@10": 0.3872531182148088, "recall@20": 0.5863055009035371, "recall@50": 0.833954370548067, "mrr": 0.7744787644787645, "pool_coverage": null, "p50_retrieve_ms": 0.0, "p95_retrieve_ms": 0.0, "git_sha": "f42c4a8eaadb3c0cc79f78c34b0e1a70534bee10", "report_path": "runs/search-esci-74-ltr-deberta-k50/report.json", "recorded_at": "2026-08-20T07:43:07.017767+00:00" }, { "run_id": "search-esci-74-ltr-deberta-lgbm-k50", "brain": "harness-local-ltr", "dataset": "search_esci_74.jsonl", "fusion": "none", "rerank": "ltr-lightgbm", "channels": [ "harness-ltr-head" ], "rank_pool": false, "rank_pool_ce": false, "n_queries": 74, "k": 50, "ndcg@10": 0.5159432058815566, "ndcg@20": 0.5658120484311862, "ndcg@50": 0.6415336394479464, "ndcg": 0.6415336394479464, "recall@10": 0.37878019077485103, "recall@20": 0.5918308018978374, "recall@50": 0.833954370548067, "mrr": 0.7098312215959275, "pool_coverage": null, "p50_retrieve_ms": 0.0, "p95_retrieve_ms": 0.0, "git_sha": "f42c4a8eaadb3c0cc79f78c34b0e1a70534bee10", "report_path": "runs/search-esci-74-ltr-deberta-lgbm-k50/report.json", "recorded_at": "2026-08-20T07:48:40.794647+00:00" }, { "run_id": "search-esci-ltr200-passages-k50", "brain": "searchbenchesciltr2", "dataset": "search_esci_ltr200.jsonl", "fusion": "rrf", "rerank": "none", "channels": [ "passages" ], "rank_pool": false, "rank_pool_ce": false, "n_queries": 170, "k": 50, "ndcg@10": 0.38688497520690734, "ndcg@20": 0.4428785087667844, "ndcg@50": 0.5191838961643084, "ndcg": 0.5191838961643084, "recall@10": 0.3668275379939918, "recall@20": 0.5506847180507305, "recall@50": 0.7435914662537475, "mrr": 0.6599425680048517, "pool_coverage": null, "p50_retrieve_ms": 76.07849999999999, "p95_retrieve_ms": 152.37659999999983, "git_sha": "f42c4a8eaadb3c0cc79f78c34b0e1a70534bee10", "report_path": "runs/search-esci-ltr200-passages-k50/report.json", "recorded_at": "2026-08-20T10:45:30.571169+00:00" }, { "run_id": "search-esci-74-ltr-deberta-train200", "brain": "harness-local-ltr", "dataset": "search_esci_74.jsonl", "fusion": "none", "rerank": "ltr-pairwise", "channels": [ "harness-ltr-head" ], "rank_pool": false, "rank_pool_ce": false, "n_queries": 74, "k": 50, "ndcg@10": 0.5436096369070966, "ndcg@20": 0.5825504636416924, "ndcg@50": 0.6525856561608427, "ndcg": 0.6525856561608427, "recall@10": 0.3911684808472767, "recall@20": 0.5850220289937974, "recall@50": 0.833954370548067, "mrr": 0.7621138996138996, "pool_coverage": null, "p50_retrieve_ms": 0.0, "p95_retrieve_ms": 0.0, "git_sha": "f42c4a8eaadb3c0cc79f78c34b0e1a70534bee10", "report_path": "runs/search-esci-74-ltr-deberta-train200/report.json", "recorded_at": "2026-08-20T10:57:34.082897+00:00" }, { "run_id": "search-esci-74-ltr-deberta-lgbm-train200", "brain": "harness-local-ltr", "dataset": "search_esci_74.jsonl", "fusion": "none", "rerank": "ltr-lightgbm", "channels": [ "harness-ltr-head" ], "rank_pool": false, "rank_pool_ce": false, "n_queries": 74, "k": 50, "ndcg@10": 0.5333470812074246, "ndcg@20": 0.5739148370361776, "ndcg@50": 0.6521013344978932, "ndcg": 0.6521013344978932, "recall@10": 0.38938710235182716, "recall@20": 0.5909398273461974, "recall@50": 0.833954370548067, "mrr": 0.7517797970384177, "pool_coverage": null, "p50_retrieve_ms": 0.0, "p95_retrieve_ms": 0.0, "git_sha": "f42c4a8eaadb3c0cc79f78c34b0e1a70534bee10", "report_path": "runs/search-esci-74-ltr-deberta-lgbm-train200/report.json", "recorded_at": "2026-08-20T11:33:18.933082+00:00" }, { "run_id": "search-esci-74-ltr-deberta-hybrid170", "brain": "harness-local-ltr", "dataset": "search_esci_74.jsonl", "fusion": "none", "rerank": "ltr-pairwise", "channels": [ "harness-ltr-head" ], "rank_pool": false, "rank_pool_ce": false, "n_queries": 74, "k": 50, "ndcg@10": 0.5304449022081842, "ndcg@20": 0.5753013243082012, "ndcg@50": 0.6452218148321592, "ndcg": 0.6452218148321592, "recall@10": 0.38518108116293304, "recall@20": 0.5882930387092911, "recall@50": 0.833954370548067, "mrr": 0.7646718146718147, "pool_coverage": null, "p50_retrieve_ms": 0.0, "p95_retrieve_ms": 0.0, "git_sha": "f42c4a8eaadb3c0cc79f78c34b0e1a70534bee10", "report_path": "runs/search-esci-74-ltr-deberta-hybrid170/report.json", "recorded_at": "2026-08-20T11:57:34.067863+00:00" }, { "run_id": "search-wands-passages-k50", "brain": "searchbenchwands", "dataset": "search_wands.jsonl", "fusion": "rrf", "rerank": "none", "channels": [ "passages" ], "rank_pool": false, "rank_pool_ce": false, "n_queries": 66, "k": 50, "ndcg@10": 0.823176135584089, "ndcg@20": 0.8120944688355884, "ndcg@50": 0.8223686140560469, "ndcg": 0.8223686140560469, "recall@10": 0.26880378393536286, "recall@20": 0.46518737110842373, "recall@50": 0.8365552576078893, "mrr": 0.9245931537598204, "pool_coverage": null, "p50_retrieve_ms": 86.595, "p95_retrieve_ms": 96.93674999999999, "git_sha": "f42c4a8eaadb3c0cc79f78c34b0e1a70534bee10", "report_path": "runs/search-wands-passages-k50/report.json", "recorded_at": "2026-08-20T13:17:40.353988+00:00" }, { "run_id": "search-wandsgraph-communities-k50", "brain": "searchbenchwandsgraph", "dataset": "search_wands.jsonl", "fusion": "rrf", "rerank": "none", "channels": [ "communities" ], "rank_pool": false, "rank_pool_ce": false, "n_queries": 66, "k": 50, "ndcg@10": 0.4094084735740818, "ndcg@20": 0.36455789387786824, "ndcg@50": 0.325224291157982, "ndcg": 0.4570937299953597, "recall@10": 0.14824773180036338, "recall@20": 0.20624528979792137, "recall@50": 0.28701452056715215, "mrr": 0.6737914862914863, "pool_coverage": null, "p50_retrieve_ms": 5609.237499999999, "p95_retrieve_ms": 11856.0285, "git_sha": "f42c4a8eaadb3c0cc79f78c34b0e1a70534bee10", "report_path": "runs/search-wandsgraph-communities-k50/report.json", "recorded_at": "2026-08-21T03:12:00.249304+00:00", "protocol": "architecture demo: isolated catalog graph on searchbenchwandsgraph; not a quality-default claim vs frozen WANDS passages 0.823 or ESCI 0.500", "claim": "architecture-demo" }, { "run_id": "search-wandsgraph-passages-k50", "brain": "searchbenchwandsgraph", "dataset": "search_wands.jsonl", "fusion": "rrf", "rerank": "none", "channels": [ "passages" ], "rank_pool": false, "rank_pool_ce": false, "n_queries": 66, "k": 50, "ndcg@10": 0.8230998576010691, "ndcg@20": 0.812056754200934, "ndcg@50": 0.8223486981324697, "ndcg": 0.8223486981324697, "recall@10": 0.26880378393536286, "recall@20": 0.46518737110842373, "recall@50": 0.8365552576078893, "mrr": 0.9245931537598204, "pool_coverage": null, "p50_retrieve_ms": 98.604, "p95_retrieve_ms": 115.12925, "git_sha": "f42c4a8eaadb3c0cc79f78c34b0e1a70534bee10", "report_path": "runs/search-wandsgraph-passages-k50/report.json", "recorded_at": "2026-08-21T03:12:36.455586+00:00", "protocol": "architecture demo: isolated catalog graph on searchbenchwandsgraph; not a quality-default claim vs frozen WANDS passages 0.823 or ESCI 0.500", "claim": "architecture-demo" }, { "run_id": "search-wandsgraph-passages-nodeid-k50", "brain": "searchbenchwandsgraph", "dataset": "search_wands.jsonl", "fusion": "rrf", "rerank": "none", "channels": [ "passages" ], "rank_pool": false, "rank_pool_ce": false, "n_queries": 66, "k": 50, "ndcg@10": 0.8230998576010691, "ndcg@20": 0.812056754200934, "ndcg@50": 0.8223486981324697, "ndcg": 0.8223486981324697, "recall@10": 0.26880378393536286, "recall@20": 0.46518737110842373, "recall@50": 0.8365552576078893, "mrr": 0.9245931537598204, "pool_coverage": null, "p50_retrieve_ms": 84.0175, "p95_retrieve_ms": 105.24199999999999, "git_sha": "f42c4a8eaadb3c0cc79f78c34b0e1a70534bee10", "report_path": "runs/search-wandsgraph-passages-nodeid-k50/report.json", "recorded_at": "2026-08-21T07:53:39.311033+00:00", "protocol": "architecture demo: isolated catalog graph on searchbenchwandsgraph; not a quality-default claim vs frozen WANDS passages 0.823 or ESCI 0.500", "claim": "architecture-demo" } ] } } }