Hugging Face's logo

Spaces:
Leon4gr45
/
brain
Running

App Files Community
1
brain / benchmarks /REPORTS.json
Leon4gr45's picture
Leon4gr45
Deploy complete BrainAPI source 81ad5ce1c9c1
eb60b6b verified
Raw History Blame Contribute Delete
95.6 kB
{
"schema_version": 2,
"updated_at": "2026-08-21T07:53:39.332214+00:00",
"description": "BrainAPI benchmark results. Top published scores across suites. Updated when a suite evaluate/report completes successfully.",
"benchmarks": {
"locomo": {
"name": "LoCoMo",
"leaderboard": [
{
"run_id": "locomo-compose-sota-conv26-v4d",
"track": "sota",
"scope": "conv-26",
"samples": [
"conv-26"
],
"n_non_adversarial": 152,
"headline_judge_accuracy": 0.9539473684210527,
"headline_judge_accuracy_pct": 95.39,
"answerable_rate": 0.9407894736842105,
"answer_model": "deepseek-v4-flash",
"judge_model": "deepseek-v4-flash",
"bench_profile": "sota",
"brain": "locomoconv26nostorm",
"git_sha": "78cbbae50ac37777ce0315f85156f7812b28248c",
"report_path": "runs/locomo-compose-sota-conv26-v4d/report.json",
"recorded_at": "2026-07-31T13:14:01.363020+00:00"
},
{
"run_id": "locomo-compose-sota-conv26-v4c",
"track": "sota",
"scope": "conv-26",
"samples": [
"conv-26"
],
"n_non_adversarial": 152,
"headline_judge_accuracy": 0.9276315789473685,
"headline_judge_accuracy_pct": 92.76,
"answerable_rate": 0.9407894736842105,
"answer_model": "deepseek-v4-flash",
"judge_model": "deepseek-v4-flash",
"bench_profile": "sota",
"brain": "locomoconv26nostorm",
"git_sha": "78cbbae50ac37777ce0315f85156f7812b28248c",
"report_path": "runs/locomo-compose-sota-conv26-v4c/report.json",
"recorded_at": "2026-07-31T13:07:26.775227+00:00"
},
{
"run_id": "locomo-compose-sota-conv26-v2",
"track": "sota",
"scope": "conv-26",
"samples": [
"conv-26"
],
"n_non_adversarial": 152,
"headline_judge_accuracy": 0.9210526315789473,
"headline_judge_accuracy_pct": 92.11,
"answerable_rate": 0.9407894736842105,
"answer_model": "deepseek-v4-flash",
"judge_model": "deepseek-v4-flash",
"bench_profile": "sota",
"brain": "locomoconv26nostorm",
"git_sha": "78cbbae50ac37777ce0315f85156f7812b28248c",
"report_path": "runs/locomo-compose-sota-conv26-v2/report.json",
"recorded_at": "2026-07-30T20:01:08.915975+00:00"
},
{
"run_id": "locomo-compose-sota-conv26-v4",
"track": "sota",
"scope": "conv-26",
"samples": [
"conv-26"
],
"n_non_adversarial": 152,
"headline_judge_accuracy": 0.9210526315789473,
"headline_judge_accuracy_pct": 92.11,
"answerable_rate": 0.9407894736842105,
"answer_model": "deepseek-v4-flash",
"judge_model": "deepseek-v4-flash",
"bench_profile": "sota",
"brain": "locomoconv26nostorm",
"git_sha": "78cbbae50ac37777ce0315f85156f7812b28248c",
"report_path": "runs/locomo-compose-sota-conv26-v4/report.json",
"recorded_at": "2026-07-31T11:28:19.452764+00:00"
},
{
"run_id": "locomo-compose-sota-conv26-v3",
"track": "sota",
"scope": "conv-26",
"samples": [
"conv-26"
],
"n_non_adversarial": 152,
"headline_judge_accuracy": 0.9013157894736842,
"headline_judge_accuracy_pct": 90.13,
"answerable_rate": 0.9407894736842105,
"answer_model": "deepseek-v4-flash",
"judge_model": "deepseek-v4-flash",
"bench_profile": "sota",
"brain": "locomoconv26nostorm",
"git_sha": "78cbbae50ac37777ce0315f85156f7812b28248c",
"report_path": "runs/locomo-compose-sota-conv26-v3/report.json",
"recorded_at": "2026-07-31T10:29:50.328215+00:00"
},
{
"run_id": "locomo-conv26-beamreg-product",
"track": "product",
"scope": "conv-26",
"samples": [
"conv-26"
],
"n_non_adversarial": 152,
"headline_judge_accuracy": 0.9013157894736842,
"headline_judge_accuracy_pct": 90.13,
"answerable_rate": 0.9473684210526315,
"answer_model": "deepseek-v4-flash",
"judge_model": "deepseek-v4-flash",
"bench_profile": "product",
"brain": "locomoconv26",
"git_sha": "78cbbae50ac37777ce0315f85156f7812b28248c",
"report_path": "runs/locomo-conv26-beamreg-product/report.json",
"recorded_at": "2026-08-01T01:58:34.693097+00:00"
},
{
"run_id": "locomo-compose-sota-conv26",
"track": "sota",
"scope": "conv-26",
"samples": [
"conv-26"
],
"n_non_adversarial": 152,
"headline_judge_accuracy": 0.8947368421052632,
"headline_judge_accuracy_pct": 89.47,
"answerable_rate": 0.9407894736842105,
"answer_model": "deepseek-v4-flash",
"judge_model": "deepseek-v4-flash",
"bench_profile": "sota",
"brain": "locomoconv26nostorm",
"git_sha": "78cbbae50ac37777ce0315f85156f7812b28248c",
"report_path": "runs/locomo-compose-sota-conv26/report.json",
"recorded_at": "2026-07-30T19:18:56.740730+00:00"
},
{
"run_id": "locomo-compose-sota-conv26-v4b",
"track": "sota",
"scope": "conv-26",
"samples": [
"conv-26"
],
"n_non_adversarial": 152,
"headline_judge_accuracy": 0.875,
"headline_judge_accuracy_pct": 87.5,
"answerable_rate": 0.9407894736842105,
"answer_model": "deepseek-v4-flash",
"judge_model": "deepseek-v4-flash",
"bench_profile": "sota",
"brain": "locomoconv26nostorm",
"git_sha": "78cbbae50ac37777ce0315f85156f7812b28248c",
"report_path": "runs/locomo-compose-sota-conv26-v4b/report.json",
"recorded_at": "2026-07-31T12:26:52.469668+00:00"
},
{
"run_id": "locomo-cat3fix-sota-conv26",
"track": "sota",
"scope": "conv-26",
"samples": [
"conv-26"
],
"n_non_adversarial": 152,
"headline_judge_accuracy": 0.868421052631579,
"headline_judge_accuracy_pct": 86.84,
"answerable_rate": 0.9407894736842105,
"answer_model": "deepseek-v4-flash",
"judge_model": "deepseek-v4-flash",
"bench_profile": "sota",
"brain": "locomoconv26nostorm",
"git_sha": "78cbbae50ac37777ce0315f85156f7812b28248c",
"report_path": "runs/locomo-cat3fix-sota-conv26/report.json",
"recorded_at": "2026-07-30T18:26:15.214852+00:00"
},
{
"run_id": "oldbrain-recheck-a",
"track": "product",
"scope": "conv-26",
"samples": [
"conv-26"
],
"n_non_adversarial": 152,
"headline_judge_accuracy": 0.8552631578947368,
"headline_judge_accuracy_pct": 85.53,
"answerable_rate": 0.9473684210526315,
"answer_model": "deepseek-v4-flash",
"judge_model": "deepseek-v4-flash",
"bench_profile": "product",
"brain": "locomoconv26",
"git_sha": "649f5e569de8894c05eec5f1f423024be5e9b6a4",
"report_path": "runs/oldbrain-recheck-a/report.json",
"recorded_at": "2026-07-28T22:05:39.768798+00:00"
},
{
"run_id": "locomo-batch-nostorm-cfgb",
"track": "product",
"scope": "conv-26",
"samples": [
"conv-26"
],
"n_non_adversarial": 152,
"headline_judge_accuracy": 0.8552631578947368,
"headline_judge_accuracy_pct": 85.53,
"answerable_rate": 0.9407894736842105,
"answer_model": "deepseek-v4-flash",
"judge_model": "deepseek-v4-flash",
"bench_profile": "product",
"brain": "locomoconv26nostorm",
"git_sha": "78cbbae50ac37777ce0315f85156f7812b28248c",
"report_path": "runs/locomo-batch-nostorm-cfgb/report.json",
"recorded_at": "2026-07-30T17:03:35.449258+00:00"
},
{
"run_id": "phase-sota-d1-conv26",
"track": "sota",
"scope": "conv-26",
"samples": [
"conv-26"
],
"n_non_adversarial": 152,
"headline_judge_accuracy": 0.8486842105263158,
"headline_judge_accuracy_pct": 84.87,
"answerable_rate": 0.9407894736842105,
"answer_model": "deepseek-v4-flash",
"judge_model": "deepseek-v4-flash",
"bench_profile": "sota",
"brain": "locomoconv26clean",
"git_sha": "649f5e569de8894c05eec5f1f423024be5e9b6a4",
"report_path": "runs/phase-sota-d1-conv26/report.json",
"recorded_at": "2026-07-29T18:51:58.400341+00:00"
},
{
"run_id": "locomo-nostorm-sota-conv26",
"track": "sota",
"scope": "conv-26",
"samples": [
"conv-26"
],
"n_non_adversarial": 152,
"headline_judge_accuracy": 0.8486842105263158,
"headline_judge_accuracy_pct": 84.87,
"answerable_rate": 0.9407894736842105,
"answer_model": "deepseek-v4-flash",
"judge_model": "deepseek-v4-flash",
"bench_profile": "sota",
"brain": "locomoconv26nostorm",
"git_sha": "78cbbae50ac37777ce0315f85156f7812b28248c",
"report_path": "runs/locomo-nostorm-sota-conv26/report.json",
"recorded_at": "2026-07-30T17:41:32.563092+00:00"
},
{
"run_id": "locomo-batch-cfgb",
"track": "product",
"scope": "conv-26",
"samples": [
"conv-26"
],
"n_non_adversarial": 152,
"headline_judge_accuracy": 0.8421052631578947,
"headline_judge_accuracy_pct": 84.21,
"answerable_rate": 0.9407894736842105,
"answer_model": "deepseek-v4-flash",
"judge_model": "deepseek-v4-flash",
"bench_profile": "product",
"brain": "locomoconv26batch",
"git_sha": "78cbbae50ac37777ce0315f85156f7812b28248c",
"report_path": "runs/locomo-batch-cfgb/report.json",
"recorded_at": "2026-07-30T15:23:27.730891+00:00"
},
{
"run_id": "clean-brain-eval-a",
"track": "product",
"scope": "conv-26",
"samples": [
"conv-26"
],
"n_non_adversarial": 152,
"headline_judge_accuracy": 0.8355263157894737,
"headline_judge_accuracy_pct": 83.55,
"answerable_rate": 0.9539473684210527,
"answer_model": "deepseek-v4-flash",
"judge_model": "deepseek-v4-flash",
"bench_profile": "product",
"brain": "locomoconv26clean",
"git_sha": "649f5e569de8894c05eec5f1f423024be5e9b6a4",
"report_path": "runs/clean-brain-eval-a/report.json",
"recorded_at": "2026-07-28T21:29:30.713806+00:00"
},
{
"run_id": "graph-stable-b",
"track": "product",
"scope": "conv-26",
"samples": [
"conv-26"
],
"n_non_adversarial": 152,
"headline_judge_accuracy": 0.8355263157894737,
"headline_judge_accuracy_pct": 83.55,
"answerable_rate": 0.9671052631578947,
"answer_model": "deepseek-v4-flash",
"judge_model": "deepseek-v4-flash",
"bench_profile": "product",
"brain": "locomoconv26clean",
"git_sha": "649f5e569de8894c05eec5f1f423024be5e9b6a4",
"report_path": "runs/graph-stable-b/report.json",
"recorded_at": "2026-07-29T10:59:22.348091+00:00"
},
{
"run_id": "clean-brain-a",
"track": "product",
"scope": "conv-26",
"samples": [
"conv-26"
],
"n_non_adversarial": 152,
"headline_judge_accuracy": 0.8289473684210527,
"headline_judge_accuracy_pct": 82.89,
"answerable_rate": 0.9473684210526315,
"answer_model": "deepseek-v4-flash",
"judge_model": "deepseek-v4-flash",
"bench_profile": "product",
"brain": "locomoconv26clean",
"git_sha": "649f5e569de8894c05eec5f1f423024be5e9b6a4",
"report_path": "runs/clean-brain-a/report.json",
"recorded_at": "2026-07-28T21:57:06.650866+00:00"
},
{
"run_id": "clean-brain-b",
"track": "product",
"scope": "conv-26",
"samples": [
"conv-26"
],
"n_non_adversarial": 152,
"headline_judge_accuracy": 0.8289473684210527,
"headline_judge_accuracy_pct": 82.89,
"answerable_rate": 0.9473684210526315,
"answer_model": "deepseek-v4-flash",
"judge_model": "deepseek-v4-flash",
"bench_profile": "product",
"brain": "locomoconv26clean",
"git_sha": "649f5e569de8894c05eec5f1f423024be5e9b6a4",
"report_path": "runs/clean-brain-b/report.json",
"recorded_at": "2026-07-28T22:13:25.651713+00:00"
},
{
"run_id": "baseline-clean-a",
"track": "product",
"scope": "conv-26",
"samples": [
"conv-26"
],
"n_non_adversarial": 152,
"headline_judge_accuracy": 0.8223684210526315,
"headline_judge_accuracy_pct": 82.24,
"answerable_rate": 0.9473684210526315,
"answer_model": "deepseek-v4-flash",
"judge_model": "deepseek-v4-flash",
"bench_profile": "product",
"brain": "locomoconv26",
"git_sha": "649f5e569de8894c05eec5f1f423024be5e9b6a4",
"report_path": "runs/baseline-clean-a/report.json",
"recorded_at": "2026-07-28T18:18:19.581660+00:00"
},
{
"run_id": "sota-locomo10-batch-compose",
"track": "sota",
"scope": "full-locomo10",
"samples": [
"conv-26",
"conv-30",
"conv-41",
"conv-42",
"conv-43",
"conv-44",
"conv-47",
"conv-48",
"conv-49",
"conv-50"
],
"n_non_adversarial": 1540,
"headline_judge_accuracy": 0.8097402597402598,
"headline_judge_accuracy_pct": 80.97,
"answerable_rate": 0.9363636363636364,
"answer_model": "deepseek-v4-flash",
"judge_model": "deepseek-v4-flash",
"bench_profile": "sota",
"brain": "locomof10c*",
"git_sha": "78cbbae50ac37777ce0315f85156f7812b28248c",
"report_path": "runs/sota-locomo10-batch-compose/report.json",
"recorded_at": "2026-07-31T13:35:58.561407+00:00"
},
{
"run_id": "phase-d1-paths-a",
"track": "product",
"scope": "conv-26",
"samples": [
"conv-26"
],
"n_non_adversarial": 152,
"headline_judge_accuracy": 0.8092105263157895,
"headline_judge_accuracy_pct": 80.92,
"answerable_rate": 0.9671052631578947,
"answer_model": "deepseek-v4-flash",
"judge_model": "deepseek-v4-flash",
"bench_profile": "product",
"brain": "locomoconv26clean",
"git_sha": "649f5e569de8894c05eec5f1f423024be5e9b6a4",
"report_path": "runs/phase-d1-paths-a/report.json",
"recorded_at": "2026-07-29T16:26:21.919353+00:00"
},
{
"run_id": "locomo-batch-score",
"track": "product",
"scope": "conv-26",
"samples": [
"conv-26"
],
"n_non_adversarial": 152,
"headline_judge_accuracy": 0.8026315789473685,
"headline_judge_accuracy_pct": 80.26,
"answerable_rate": 0.9407894736842105,
"answer_model": "deepseek-v4-flash",
"judge_model": "deepseek-v4-flash",
"bench_profile": "product",
"brain": "locomoconv26batch",
"git_sha": "78cbbae50ac37777ce0315f85156f7812b28248c",
"report_path": "runs/locomo-batch-score/report.json",
"recorded_at": "2026-07-30T14:59:39.743019+00:00"
}
]
},
"longmemeval": {
"name": "LongMemEval",
"leaderboard": []
},
"beam": {
"name": "BEAM",
"leaderboard": [
{
"run_id": "beam-1m-1-product-v3c-compose",
"track": "product",
"bench_profile": "product",
"scope": "1M/1",
"size": "1M",
"samples": [
"1M/1"
],
"n_questions": 20,
"headline_score": 0.7897396010333467,
"headline_score_pct": 78.97,
"per_ability": {
"abstention": 0.5,
"contradiction_resolution": 0.6875,
"event_ordering": 0.42170156588902286,
"information_extraction": 1.0,
"instruction_following": 1.0,
"knowledge_update": 1.0,
"multi_session_reasoning": 0.9375,
"preference_following": 0.875,
"summarization": 0.4756944444444444,
"temporal_reasoning": 1.0
},
"answer_model": "deepseek-v4-flash",
"judge_model": "deepseek-v4-flash",
"brain": "beam1m1clean",
"git_sha": "78cbbae50ac37777ce0315f85156f7812b28248c",
"report_path": "runs/beam-1m-1-product-v3c-compose/report.json",
"recorded_at": "2026-08-03T09:15:33.803818+00:00"
},
{
"run_id": "beam-100k-1-clean-ordv1",
"track": "product",
"bench_profile": "product",
"scope": "100K/1",
"size": "100K",
"samples": [
"100K/1"
],
"n_questions": 20,
"headline_score": 0.7887500000000001,
"headline_score_pct": 78.88,
"per_ability": {
"abstention": 1.0,
"contradiction_resolution": 0.625,
"event_ordering": 0.7875,
"information_extraction": 1.0,
"instruction_following": 0.75,
"knowledge_update": 1.0,
"multi_session_reasoning": 0.625,
"preference_following": 1.0,
"summarization": 0.6000000000000001,
"temporal_reasoning": 0.5
},
"answer_model": "deepseek-v4-flash",
"judge_model": "deepseek-v4-flash",
"brain": "beam100k1clean",
"git_sha": "78cbbae50ac37777ce0315f85156f7812b28248c",
"report_path": "runs/beam-100k-1-clean-ordv1/report.json",
"recorded_at": "2026-08-01T02:00:50.310070+00:00"
},
{
"run_id": "beam-100k-1-clean-product",
"track": "product",
"bench_profile": "product",
"scope": "100K/1",
"size": "100K",
"samples": [
"100K/1"
],
"n_questions": 20,
"headline_score": 0.7398223304703363,
"headline_score_pct": 73.98,
"per_ability": {
"abstention": 0.5,
"contradiction_resolution": 0.6875,
"event_ordering": 0.1357233047033631,
"information_extraction": 0.75,
"instruction_following": 1.0,
"knowledge_update": 1.0,
"multi_session_reasoning": 0.875,
"preference_following": 1.0,
"summarization": 0.7,
"temporal_reasoning": 0.75
},
"answer_model": "deepseek-v4-flash",
"judge_model": "deepseek-v4-flash",
"brain": "beam100k1clean",
"git_sha": "78cbbae50ac37777ce0315f85156f7812b28248c",
"report_path": "runs/beam-100k-1-clean-product/report.json",
"recorded_at": "2026-08-01T02:00:51.059632+00:00"
},
{
"run_id": "beam-1m-1-product-v2",
"track": "product",
"bench_profile": "product",
"scope": "1M/1",
"size": "1M",
"samples": [
"1M/1"
],
"n_questions": 20,
"headline_score": 0.7014047619826247,
"headline_score_pct": 70.14,
"per_ability": {
"abstention": 0.5,
"contradiction_resolution": 0.4375,
"event_ordering": 0.2883531753818027,
"information_extraction": 1.0,
"instruction_following": 1.0,
"knowledge_update": 1.0,
"multi_session_reasoning": 0.625,
"preference_following": 0.75,
"summarization": 0.4131944444444444,
"temporal_reasoning": 1.0
},
"answer_model": "deepseek-v4-flash",
"judge_model": "deepseek-v4-flash",
"brain": "beam1m1clean",
"git_sha": "78cbbae50ac37777ce0315f85156f7812b28248c",
"report_path": "runs/beam-1m-1-product-v2/report.json",
"recorded_at": "2026-08-03T07:58:27.006989+00:00"
},
{
"run_id": "beam-1m-1-product-v3b",
"track": "product",
"bench_profile": "product",
"scope": "1M/1",
"size": "1M",
"samples": [
"1M/1"
],
"n_questions": 20,
"headline_score": 0.6497747205349994,
"headline_score_pct": 64.98,
"per_ability": {
"abstention": 0.5,
"contradiction_resolution": 0.5625,
"event_ordering": 0.4213583164611052,
"information_extraction": 1.0,
"instruction_following": 1.0,
"knowledge_update": 0.5,
"multi_session_reasoning": 0.75,
"preference_following": 0.875,
"summarization": 0.3888888888888889,
"temporal_reasoning": 0.5
},
"answer_model": "deepseek-v4-flash",
"judge_model": "deepseek-v4-flash",
"brain": "beam1m1clean",
"git_sha": "78cbbae50ac37777ce0315f85156f7812b28248c",
"report_path": "runs/beam-1m-1-product-v3b/report.json",
"recorded_at": "2026-08-03T09:14:47.079177+00:00"
},
{
"run_id": "beam-100k-1-sota",
"track": "sota",
"bench_profile": "sota",
"scope": "100K/1",
"size": "100K",
"samples": [
"100K/1"
],
"n_questions": 20,
"headline_score": 0.6483928571428572,
"headline_score_pct": 64.84,
"per_ability": {
"abstention": 0.0,
"contradiction_resolution": 0.625,
"event_ordering": 0.1339285714285714,
"information_extraction": 0.75,
"instruction_following": 0.75,
"knowledge_update": 1.0,
"multi_session_reasoning": 0.875,
"preference_following": 1.0,
"summarization": 0.6,
"temporal_reasoning": 0.75
},
"answer_model": "deepseek-v4-flash",
"judge_model": "deepseek-v4-flash",
"brain": "beam100k1",
"git_sha": "78cbbae50ac37777ce0315f85156f7812b28248c",
"report_path": "runs/beam-100k-1-sota/report.json",
"recorded_at": "2026-08-01T02:01:11.568901+00:00"
},
{
"run_id": "beam-100k-1-clean-sota",
"track": "sota",
"bench_profile": "sota",
"scope": "100K/1",
"size": "100K",
"samples": [
"100K/1"
],
"n_questions": 20,
"headline_score": 0.6245833333333334,
"headline_score_pct": 62.46,
"per_ability": {
"abstention": 0.0,
"contradiction_resolution": 0.875,
"event_ordering": 0.1458333333333333,
"information_extraction": 0.5,
"instruction_following": 0.75,
"knowledge_update": 1.0,
"multi_session_reasoning": 0.625,
"preference_following": 1.0,
"summarization": 0.6000000000000001,
"temporal_reasoning": 0.75
},
"answer_model": "deepseek-v4-flash",
"judge_model": "deepseek-v4-flash",
"brain": "beam100k1clean",
"git_sha": "78cbbae50ac37777ce0315f85156f7812b28248c",
"report_path": "runs/beam-100k-1-clean-sota/report.json",
"recorded_at": "2026-08-01T02:01:12.316230+00:00"
},
{
"run_id": "beam-100k-1-product-prompts",
"track": "product",
"bench_profile": "product",
"scope": "100K/1",
"size": "100K",
"samples": [
"100K/1"
],
"n_questions": 20,
"headline_score": 0.6042261904761904,
"headline_score_pct": 60.42,
"per_ability": {
"abstention": 0.0,
"contradiction_resolution": 0.5,
"event_ordering": 0.1339285714285714,
"information_extraction": 0.9583333333333333,
"instruction_following": 0.75,
"knowledge_update": 1.0,
"multi_session_reasoning": 0.75,
"preference_following": 0.75,
"summarization": 0.7,
"temporal_reasoning": 0.5
},
"answer_model": "deepseek-v4-flash",
"judge_model": "deepseek-v4-flash",
"brain": "beam100k1",
"git_sha": "78cbbae50ac37777ce0315f85156f7812b28248c",
"report_path": "runs/beam-100k-1-product-prompts/report.json",
"recorded_at": "2026-08-01T02:01:13.074254+00:00"
},
{
"run_id": "beam-100k-1-configb",
"track": "product",
"bench_profile": "product",
"scope": "100K/1",
"size": "100K",
"samples": [
"100K/1"
],
"n_questions": 20,
"headline_score": 0.559333376188726,
"headline_score_pct": 55.93,
"per_ability": {
"abstention": 0.25,
"contradiction_resolution": 0.5,
"event_ordering": 0.14333376188726035,
"information_extraction": 0.5,
"instruction_following": 0.75,
"knowledge_update": 1.0,
"multi_session_reasoning": 0.625,
"preference_following": 0.75,
"summarization": 0.7,
"temporal_reasoning": 0.375
},
"answer_model": "deepseek-v4-flash",
"judge_model": "deepseek-v4-flash",
"brain": "beam100k1",
"git_sha": "78cbbae50ac37777ce0315f85156f7812b28248c",
"report_path": "runs/beam-100k-1-configb/report.json",
"recorded_at": "2026-08-01T02:01:13.832770+00:00"
},
{
"run_id": "beam-100k-1",
"track": "product",
"bench_profile": "product",
"scope": "100K/1",
"size": "100K",
"samples": [
"100K/1"
],
"n_questions": 20,
"headline_score": 0.5558928571428572,
"headline_score_pct": 55.59,
"per_ability": {
"abstention": 0.5,
"contradiction_resolution": 0.5,
"event_ordering": 0.1339285714285714,
"information_extraction": 0.5,
"instruction_following": 0.5,
"knowledge_update": 1.0,
"multi_session_reasoning": 0.625,
"preference_following": 0.75,
"summarization": 0.55,
"temporal_reasoning": 0.5
},
"answer_model": "deepseek-v4-flash",
"judge_model": "deepseek-v4-flash",
"brain": "beam100k1",
"git_sha": "78cbbae50ac37777ce0315f85156f7812b28248c",
"report_path": "runs/beam-100k-1/report.json",
"recorded_at": "2026-08-01T02:01:14.597463+00:00"
},
{
"run_id": "beam-1m-1-clean-product",
"track": "product",
"bench_profile": "product",
"scope": "1M/1",
"size": "1M",
"samples": [
"1M/1"
],
"n_questions": 20,
"headline_score": 0.5520385817869101,
"headline_score_pct": 55.2,
"per_ability": {
"abstention": 0.5,
"contradiction_resolution": 0.375,
"event_ordering": 0.35024692898021254,
"information_extraction": 1.0,
"instruction_following": 0.5,
"knowledge_update": 0.5,
"multi_session_reasoning": 0.625,
"preference_following": 0.75,
"summarization": 0.4201388888888889,
"temporal_reasoning": 0.5
},
"answer_model": "deepseek-v4-flash",
"judge_model": "deepseek-v4-flash",
"brain": "beam1m1clean",
"git_sha": "78cbbae50ac37777ce0315f85156f7812b28248c",
"report_path": "runs/beam-1m-1-clean-product/report.json",
"recorded_at": "2026-08-03T06:59:14.016975+00:00"
}
]
},
"recsys": {
"name": "RecSys (event-KG recommend)",
"leaderboard": [
{
"run_id": "recsys-20260805T200653Z",
"brain": "demorecsys",
"n_users": 20,
"hit_rate@10": 0.1,
"hit_rate@20": 0.1,
"recall@10": 0.1,
"recall@20": 0.1,
"mean_intra_list_diversity": 0.20536531279178338,
"mean_type_coverage": 2.0,
"git_sha": "cf61c99b809730cc9eb35db3532e018bdd038a55",
"report_path": "runs/recsys-20260805T200653Z/report.json",
"recorded_at": "2026-08-05T20:07:44.707984+00:00"
}
]
},
"search": {
"name": "Search (hybrid BM25 + dense)",
"representative_run_id": "search-wands-passages-k50",
"representative_scope": "WANDS frozen slice: 66 queries / 2000 documents",
"leaderboard": [
{
"run_id": "search-20260818T111250Z",
"brain": "searchbenchsmoke",
"dataset": "search_toy.jsonl",
"fusion": "rrf",
"rerank": "none",
"n_queries": 10,
"ndcg@10": 1.0,
"recall@10": 1.0,
"mrr": 1.0,
"p50_retrieve_ms": 25.1575,
"p95_retrieve_ms": 27.67875,
"git_sha": "f42c4a8eaadb3c0cc79f78c34b0e1a70534bee10",
"report_path": "runs/search-20260818T111250Z/report.json",
"recorded_at": "2026-08-18T11:13:57.784280+00:00"
},
{
"run_id": "search-toy-cc",
"brain": "searchbenchsmoke",
"dataset": "search_toy.jsonl",
"fusion": "cc",
"rerank": "none",
"n_queries": 10,
"ndcg@10": 1.0,
"recall@10": 1.0,
"mrr": 1.0,
"p50_retrieve_ms": 24.2875,
"p95_retrieve_ms": 41.931949999999986,
"git_sha": "f42c4a8eaadb3c0cc79f78c34b0e1a70534bee10",
"report_path": "runs/search-toy-cc/report.json",
"recorded_at": "2026-08-18T11:15:19.039999+00:00"
},
{
"run_id": "search-esci-slice-allcols",
"brain": "searchbenchesci20",
"dataset": "search_esci_slice.jsonl",
"fusion": "rrf",
"rerank": "none",
"channels": [
"passages",
"entities",
"communities"
],
"n_queries": 11,
"ndcg@10": 0.6815199154995802,
"recall@10": 0.4656565656565656,
"mrr": 0.8484848484848485,
"p50_retrieve_ms": 108.159,
"p95_retrieve_ms": 129.32150000000001,
"git_sha": "f42c4a8eaadb3c0cc79f78c34b0e1a70534bee10",
"report_path": "runs/search-esci-slice-allcols/report.json",
"recorded_at": "2026-08-18T13:02:36.140118+00:00"
},
{
"run_id": "search-esci-slice-passages",
"brain": "searchbenchesci20",
"dataset": "search_esci_slice.jsonl",
"fusion": "rrf",
"rerank": "none",
"channels": [
"passages"
],
"n_queries": 11,
"ndcg@10": 0.7579469533596556,
"recall@10": 0.5106060606060606,
"mrr": 0.8484848484848485,
"p50_retrieve_ms": 29.279,
"p95_retrieve_ms": 153.97,
"git_sha": "f42c4a8eaadb3c0cc79f78c34b0e1a70534bee10",
"report_path": "runs/search-esci-slice-passages/report.json",
"recorded_at": "2026-08-18T13:02:47.533040+00:00"
},
{
"run_id": "search-esci-slice-entities",
"brain": "searchbenchesci20",
"dataset": "search_esci_slice.jsonl",
"fusion": "rrf",
"rerank": "none",
"channels": [
"entities"
],
"n_queries": 11,
"ndcg@10": 0.5323459350114446,
"recall@10": 0.4151515151515151,
"mrr": 0.6673553719008264,
"p50_retrieve_ms": 52.75,
"p95_retrieve_ms": 75.479,
"git_sha": "f42c4a8eaadb3c0cc79f78c34b0e1a70534bee10",
"report_path": "runs/search-esci-slice-entities/report.json",
"recorded_at": "2026-08-18T13:31:39.862006+00:00"
},
{
"run_id": "search-esci-slice-communities",
"brain": "searchbenchesci20",
"dataset": "search_esci_slice.jsonl",
"fusion": "rrf",
"rerank": "none",
"channels": [
"communities"
],
"n_queries": 11,
"ndcg@10": 0.0,
"recall@10": 0.0,
"mrr": 0.0,
"p50_retrieve_ms": 51.536,
"p95_retrieve_ms": 58.432500000000005,
"git_sha": "f42c4a8eaadb3c0cc79f78c34b0e1a70534bee10",
"report_path": "runs/search-esci-slice-communities/report.json",
"recorded_at": "2026-08-18T13:31:42.972981+00:00"
},
{
"run_id": "search-esci-slice-pe",
"brain": "searchbenchesci20",
"dataset": "search_esci_slice.jsonl",
"fusion": "rrf",
"rerank": "none",
"channels": [
"passages",
"entities"
],
"n_queries": 11,
"ndcg@10": 0.6815199154995802,
"recall@10": 0.4656565656565656,
"mrr": 0.8484848484848485,
"p50_retrieve_ms": 74.312,
"p95_retrieve_ms": 91.006,
"git_sha": "f42c4a8eaadb3c0cc79f78c34b0e1a70534bee10",
"report_path": "runs/search-esci-slice-pe/report.json",
"recorded_at": "2026-08-18T13:31:46.700938+00:00"
},
{
"run_id": "search-esci-slice-entities-after",
"brain": "searchbenchesci20",
"dataset": "search_esci_slice.jsonl",
"fusion": "rrf",
"rerank": "none",
"channels": [
"entities"
],
"n_queries": 11,
"ndcg@10": 0.6549492067418874,
"recall@10": 0.4818181818181818,
"mrr": 0.7568181818181817,
"p50_retrieve_ms": 51.071,
"p95_retrieve_ms": 74.023,
"git_sha": "f42c4a8eaadb3c0cc79f78c34b0e1a70534bee10",
"report_path": "runs/search-esci-slice-entities-after/report.json",
"recorded_at": "2026-08-18T13:36:54.694299+00:00"
},
{
"run_id": "search-esci-slice-communities-after",
"brain": "searchbenchesci20",
"dataset": "search_esci_slice.jsonl",
"fusion": "rrf",
"rerank": "none",
"channels": [
"communities"
],
"n_queries": 11,
"ndcg@10": 0.4776363794553365,
"recall@10": 0.35000000000000003,
"mrr": 0.8484848484848485,
"p50_retrieve_ms": 129.202,
"p95_retrieve_ms": 152.889,
"git_sha": "f42c4a8eaadb3c0cc79f78c34b0e1a70534bee10",
"report_path": "runs/search-esci-slice-communities-after/report.json",
"recorded_at": "2026-08-18T13:39:09.162227+00:00"
},
{
"run_id": "search-esci-slice-passages-smoke",
"brain": "searchbenchesci20",
"dataset": "search_esci_slice.jsonl",
"fusion": "rrf",
"rerank": "none",
"channels": [
"passages"
],
"n_queries": 11,
"ndcg@10": 0.7579469533596556,
"recall@10": 0.5106060606060606,
"mrr": 0.8484848484848485,
"p50_retrieve_ms": 27.735,
"p95_retrieve_ms": 32.483000000000004,
"git_sha": "f42c4a8eaadb3c0cc79f78c34b0e1a70534bee10",
"report_path": "runs/search-esci-slice-passages-smoke/report.json",
"recorded_at": "2026-08-18T13:37:03.038349+00:00"
},
{
"run_id": "search-esci-slice-pec-after",
"brain": "searchbenchesci20",
"dataset": "search_esci_slice.jsonl",
"fusion": "rrf",
"rerank": "none",
"channels": [
"passages",
"entities",
"communities"
],
"n_queries": 11,
"ndcg@10": 0.6526549564168582,
"recall@10": 0.496969696969697,
"mrr": 0.816017316017316,
"p50_retrieve_ms": 183.946,
"p95_retrieve_ms": 216.89249999999998,
"git_sha": "f42c4a8eaadb3c0cc79f78c34b0e1a70534bee10",
"report_path": "runs/search-esci-slice-pec-after/report.json",
"recorded_at": "2026-08-18T13:39:52.408131+00:00"
},
{
"run_id": "search-esci-slice-passages-ce",
"brain": "searchbenchesci20",
"dataset": "search_esci_slice.jsonl",
"fusion": "rrf",
"rerank": "plugin:cross-encoder",
"channels": [
"passages"
],
"rank_pool": false,
"rank_pool_ce": false,
"n_queries": 11,
"ndcg@10": 0.7375861975343849,
"ndcg@20": null,
"ndcg": null,
"recall@10": 0.5106060606060606,
"recall@20": 0.8469696969696969,
"mrr": 0.8030303030303031,
"p50_retrieve_ms": 30.029,
"p95_retrieve_ms": 76.32499999999999,
"git_sha": "f42c4a8eaadb3c0cc79f78c34b0e1a70534bee10",
"report_path": "runs/search-esci-slice-passages-ce/report.json",
"recorded_at": "2026-08-18T14:20:28.184907+00:00"
},
{
"run_id": "search-esci-slice-passages-pool",
"brain": "searchbenchesci20",
"dataset": "search_esci_slice.jsonl",
"fusion": "rrf",
"rerank": "none",
"channels": [
"passages"
],
"rank_pool": true,
"n_queries": 11,
"ndcg@10": 0.7884517684883318,
"ndcg@20": 0.8114424261982733,
"ndcg": 0.8137804024940007,
"recall@10": 0.5348484848484848,
"recall@20": 0.8469696969696969,
"mrr": 0.9090909090909091,
"p50_retrieve_ms": 22.721,
"p95_retrieve_ms": 75.3825,
"git_sha": "f42c4a8eaadb3c0cc79f78c34b0e1a70534bee10",
"report_path": "runs/search-esci-slice-passages-pool/report.json",
"recorded_at": "2026-08-18T14:18:44.171034+00:00"
},
{
"run_id": "search-esci-slice-ce-pool",
"brain": "searchbenchesci20",
"dataset": "search_esci_slice.jsonl",
"fusion": "none",
"rerank": "plugin:cross-encoder",
"channels": [
"rank-pool-ce"
],
"rank_pool": true,
"rank_pool_ce": true,
"n_queries": 11,
"ndcg@10": 0.5976277406374534,
"ndcg@20": 0.668518640484444,
"ndcg": 0.7862048091602294,
"recall@10": 0.398637349324982,
"recall@20": 0.6598423817505625,
"mrr": 0.8333333333333333,
"p50_retrieve_ms": 0.0,
"p95_retrieve_ms": 0.0,
"git_sha": "f42c4a8eaadb3c0cc79f78c34b0e1a70534bee10",
"report_path": "runs/search-esci-slice-ce-pool/report.json",
"recorded_at": "2026-08-18T14:20:28.034427+00:00"
},
{
"run_id": "search-esci-74-ce-pool",
"brain": "searchbenchesci74",
"dataset": "search_esci_74.jsonl",
"fusion": "none",
"rerank": "plugin:cross-encoder",
"channels": [
"rank-pool-ce"
],
"rank_pool": true,
"rank_pool_ce": true,
"n_queries": 74,
"ndcg@10": 0.5492314171926193,
"ndcg@20": 0.6311374744057928,
"ndcg": 0.7174501167408996,
"recall@10": 0.4907400145298616,
"recall@20": 0.7577102510384188,
"mrr": 0.8306570306570307,
"p50_retrieve_ms": 0.0,
"p95_retrieve_ms": 0.0,
"git_sha": "f42c4a8eaadb3c0cc79f78c34b0e1a70534bee10",
"report_path": "runs/search-esci-74-ce-pool/report.json",
"recorded_at": "2026-08-18T14:29:17.114211+00:00"
},
{
"run_id": "search-esci-74-ce-pool-ft",
"brain": "searchbenchesci74",
"dataset": "search_esci_74.jsonl",
"fusion": "none",
"rerank": "plugin:cross-encoder",
"channels": [
"rank-pool-ce"
],
"rank_pool": true,
"rank_pool_ce": true,
"n_queries": 74,
"ndcg@10": 0.5832632846066277,
"ndcg@20": 0.6690889170924552,
"ndcg": 0.7451408010463653,
"recall@10": 0.4893954738298456,
"recall@20": 0.7641337775411228,
"mrr": 0.8932519057519057,
"p50_retrieve_ms": 0.0,
"p95_retrieve_ms": 0.0,
"git_sha": "f42c4a8eaadb3c0cc79f78c34b0e1a70534bee10",
"report_path": "runs/search-esci-74-ce-pool-ft/report.json",
"recorded_at": "2026-08-18T14:32:48.672213+00:00"
},
{
"run_id": "search-esci-74-passages",
"brain": "searchbenchesci74",
"dataset": "search_esci_74.jsonl",
"fusion": "rrf",
"rerank": "none",
"channels": [
"passages"
],
"rank_pool": false,
"rank_pool_ce": false,
"n_queries": 74,
"ndcg@10": 0.49528253069041633,
"ndcg@20": 0.5490094202707876,
"ndcg": 0.5490094202707876,
"recall@10": 0.3769307112873572,
"recall@20": 0.5920965629425753,
"mrr": 0.7580598455598455,
"p50_retrieve_ms": 70.9225,
"p95_retrieve_ms": 94.17495,
"git_sha": "f42c4a8eaadb3c0cc79f78c34b0e1a70534bee10",
"report_path": "runs/search-esci-74-passages/report.json",
"recorded_at": "2026-08-18T15:38:50.867049+00:00"
},
{
"run_id": "search-esci-74-passages-pool",
"brain": "searchbenchesci74",
"dataset": "search_esci_74.jsonl",
"fusion": "rrf",
"rerank": "none",
"channels": [
"passages"
],
"rank_pool": true,
"rank_pool_ce": false,
"n_queries": 74,
"ndcg@10": 0.5458756376131827,
"ndcg@20": 0.5785048208487761,
"ndcg": 0.6013933046523523,
"recall@10": 0.42064799804369324,
"recall@20": 0.5920965629425753,
"mrr": 0.8254504504504505,
"p50_retrieve_ms": 65.447,
"p95_retrieve_ms": 91.44999999999999,
"git_sha": "f42c4a8eaadb3c0cc79f78c34b0e1a70534bee10",
"report_path": "runs/search-esci-74-passages-pool/report.json",
"recorded_at": "2026-08-18T15:41:37.970033+00:00"
},
{
"run_id": "search-esci-slice-passages-gate",
"brain": "searchbenchesci20",
"dataset": "search_esci_slice.jsonl",
"fusion": "rrf",
"rerank": "none",
"channels": [
"passages"
],
"rank_pool": false,
"rank_pool_ce": false,
"n_queries": 11,
"ndcg@10": 0.7579469533596556,
"ndcg@20": 0.7962544743266965,
"ndcg": 0.7962544743266965,
"recall@10": 0.5106060606060606,
"recall@20": 0.8469696969696969,
"mrr": 0.8484848484848485,
"p50_retrieve_ms": 25.549,
"p95_retrieve_ms": 41.532,
"git_sha": "f42c4a8eaadb3c0cc79f78c34b0e1a70534bee10",
"report_path": "runs/search-esci-slice-passages-gate/report.json",
"recorded_at": "2026-08-18T16:47:11.911155+00:00"
},
{
"run_id": "search-esci-slice-pec-lists",
"brain": "searchbenchesci20",
"dataset": "search_esci_slice.jsonl",
"fusion": "rrf",
"rerank": "none",
"channels": [
"passages",
"entities",
"communities"
],
"rank_pool": false,
"rank_pool_ce": false,
"n_queries": 11,
"ndcg@10": 0.6526549564168582,
"ndcg@20": 0.6261389422887412,
"ndcg": 0.683063799169122,
"recall@10": 0.496969696969697,
"recall@20": 0.6530303030303031,
"mrr": 0.816017316017316,
"p50_retrieve_ms": 160.231,
"p95_retrieve_ms": 198.8395,
"git_sha": "f42c4a8eaadb3c0cc79f78c34b0e1a70534bee10",
"report_path": "runs/search-esci-slice-pec-lists/report.json",
"recorded_at": "2026-08-18T16:47:26.669145+00:00"
},
{
"run_id": "search-esci-slice-entities-neighbors",
"brain": "searchbenchesci20",
"dataset": "search_esci_slice.jsonl",
"fusion": "rrf",
"rerank": "none",
"channels": [
"entities"
],
"rank_pool": false,
"rank_pool_ce": false,
"n_queries": 11,
"ndcg@10": 0.6549492067418874,
"ndcg@20": 0.7173885785786568,
"ndcg": 0.7173885785786568,
"recall@10": 0.4818181818181818,
"recall@20": 0.8217171717171716,
"mrr": 0.7568181818181817,
"p50_retrieve_ms": 223.289,
"p95_retrieve_ms": 271.031,
"git_sha": "f42c4a8eaadb3c0cc79f78c34b0e1a70534bee10",
"report_path": "runs/search-esci-slice-entities-neighbors/report.json",
"recorded_at": "2026-08-18T16:48:04.631525+00:00"
},
{
"run_id": "search-esci-74-passages-control",
"brain": "searchbenchesci74",
"dataset": "search_esci_74.jsonl",
"fusion": "rrf",
"rerank": "none",
"channels": [
"passages"
],
"rank_pool": false,
"rank_pool_ce": false,
"n_queries": 74,
"ndcg@10": 0.49528253069041633,
"ndcg@20": 0.5490094202707876,
"ndcg": 0.5490094202707876,
"recall@10": 0.3769307112873572,
"recall@20": 0.5920965629425753,
"mrr": 0.7580598455598455,
"p50_retrieve_ms": 61.7115,
"p95_retrieve_ms": 91.52104999999995,
"git_sha": "f42c4a8eaadb3c0cc79f78c34b0e1a70534bee10",
"report_path": "runs/search-esci-74-passages-control/report.json",
"recorded_at": "2026-08-18T17:13:00.463193+00:00"
},
{
"run_id": "search-esci-74-ce-pool-l12-4class",
"brain": "searchbenchesci74",
"dataset": "search_esci_74.jsonl",
"fusion": "none",
"rerank": "4class-weighted-ce",
"channels": [
"rank-pool-4class"
],
"rank_pool": true,
"rank_pool_ce": true,
"n_queries": 74,
"ndcg@10": 0.5968432958924662,
"ndcg@20": 0.6782580445181766,
"ndcg": 0.7467109659912253,
"recall@10": 0.5088965287461584,
"recall@20": 0.7673275730894211,
"mrr": 0.8351464398761695,
"p50_retrieve_ms": 0.0,
"p95_retrieve_ms": 0.0,
"git_sha": "f42c4a8eaadb3c0cc79f78c34b0e1a70534bee10",
"report_path": "runs/search-esci-74-ce-pool-l12-4class/report.json",
"recorded_at": "2026-08-18T17:44:17.593025+00:00"
},
{
"run_id": "search-esci-74-ce-pool-4class",
"brain": "searchbenchesci74",
"dataset": "search_esci_74.jsonl",
"fusion": "none",
"rerank": "4class-weighted-ce",
"channels": [
"rank-pool-4class"
],
"rank_pool": true,
"rank_pool_ce": true,
"n_queries": 74,
"ndcg@10": 0.5826375707951627,
"ndcg@20": 0.6696127031036031,
"ndcg": 0.7419804620261266,
"recall@10": 0.49798997460620914,
"recall@20": 0.7668403956654921,
"mrr": 0.8317567567567568,
"p50_retrieve_ms": 0.0,
"p95_retrieve_ms": 0.0,
"git_sha": "f42c4a8eaadb3c0cc79f78c34b0e1a70534bee10",
"report_path": "runs/search-esci-74-ce-pool-4class/report.json",
"recorded_at": "2026-08-18T17:54:14.073711+00:00"
},
{
"run_id": "search-esci-74-ce-pool-4class-nowt",
"brain": "searchbenchesci74",
"dataset": "search_esci_74.jsonl",
"fusion": "none",
"rerank": "4class-weighted-ce",
"channels": [
"rank-pool-4class"
],
"rank_pool": true,
"rank_pool_ce": true,
"n_queries": 74,
"ndcg@10": 0.6096378704103984,
"ndcg@20": 0.6881968860231006,
"ndcg": 0.7594213054854562,
"recall@10": 0.5249760000665886,
"recall@20": 0.7631438693842529,
"mrr": 0.8530036338859869,
"p50_retrieve_ms": 0.0,
"p95_retrieve_ms": 0.0,
"git_sha": "f42c4a8eaadb3c0cc79f78c34b0e1a70534bee10",
"report_path": "runs/search-esci-74-ce-pool-4class-nowt/report.json",
"recorded_at": "2026-08-18T18:20:20.750483+00:00"
},
{
"run_id": "search-esci-74-ce-pool-4class-e2",
"brain": "searchbenchesci74",
"dataset": "search_esci_74.jsonl",
"fusion": "none",
"rerank": "4class-weighted-ce",
"channels": [
"rank-pool-4class"
],
"rank_pool": true,
"rank_pool_ce": true,
"n_queries": 74,
"ndcg@10": 0.6169746541917066,
"ndcg@20": 0.6945278030601721,
"ndcg": 0.7652624576380807,
"recall@10": 0.5214445444747297,
"recall@20": 0.7719306369596903,
"mrr": 0.8525418275418275,
"p50_retrieve_ms": 0.0,
"p95_retrieve_ms": 0.0,
"git_sha": "f42c4a8eaadb3c0cc79f78c34b0e1a70534bee10",
"report_path": "runs/search-esci-74-ce-pool-4class-e2/report.json",
"recorded_at": "2026-08-18T18:48:00.939749+00:00"
},
{
"run_id": "search-esci-74-passages-k50",
"brain": "searchbenchesci74",
"dataset": "search_esci_74.jsonl",
"fusion": "rrf",
"rerank": "none",
"channels": [
"passages"
],
"rank_pool": false,
"rank_pool_ce": false,
"n_queries": 74,
"k": 50,
"ndcg@10": 0.4998984443226263,
"ndcg@20": 0.5522842574567189,
"ndcg@50": 0.6378451547901237,
"ndcg": 0.6378451547901237,
"recall@10": 0.378732513089159,
"recall@20": 0.589865760711773,
"recall@50": 0.833954370548067,
"mrr": 0.7645849420849421,
"pool_coverage": null,
"p50_retrieve_ms": 60.091,
"p95_retrieve_ms": 88.02255,
"git_sha": "f42c4a8eaadb3c0cc79f78c34b0e1a70534bee10",
"report_path": "runs/search-esci-74-passages-k50/report.json",
"recorded_at": "2026-08-18T20:31:55.998500+00:00"
},
{
"run_id": "search-esci-74-passages-k100",
"brain": "searchbenchesci74",
"dataset": "search_esci_74.jsonl",
"fusion": "rrf",
"rerank": "none",
"channels": [
"passages"
],
"rank_pool": false,
"rank_pool_ce": false,
"n_queries": 74,
"k": 100,
"ndcg@10": 0.4998055248060819,
"ndcg@20": 0.5523315287868084,
"ndcg@50": 0.6375525242924566,
"ndcg": 0.6564185982755066,
"recall@10": 0.378732513089159,
"recall@20": 0.5913672622132746,
"recall@50": 0.830951367545064,
"mrr": 0.76497727997728,
"pool_coverage": null,
"p50_retrieve_ms": 57.553,
"p95_retrieve_ms": 81.81569999999999,
"git_sha": "f42c4a8eaadb3c0cc79f78c34b0e1a70534bee10",
"report_path": "runs/search-esci-74-passages-k100/report.json",
"recorded_at": "2026-08-18T20:32:27.553496+00:00"
},
{
"run_id": "search-esci-74-passages-k50-pool",
"brain": "searchbenchesci74",
"dataset": "search_esci_74.jsonl",
"fusion": "rrf",
"rerank": "none",
"channels": [
"passages"
],
"rank_pool": true,
"rank_pool_ce": false,
"n_queries": 74,
"k": 50,
"ndcg@10": 0.5668024300832123,
"ndcg@20": 0.624102711461205,
"ndcg@50": 0.6826188564511254,
"ndcg": 0.6947261855882534,
"recall@10": 0.4507852781809733,
"recall@20": 0.6686543312372134,
"recall@50": 0.8324528690465655,
"mrr": 0.8271396396396397,
"pool_coverage": 0.778222315718655,
"p50_retrieve_ms": 60.1315,
"p95_retrieve_ms": 86.49954999999997,
"git_sha": "f42c4a8eaadb3c0cc79f78c34b0e1a70534bee10",
"report_path": "runs/search-esci-74-passages-k50-pool/report.json",
"recorded_at": "2026-08-18T20:33:02.203247+00:00"
},
{
"run_id": "search-esci-74-passages-k100-pool",
"brain": "searchbenchesci74",
"dataset": "search_esci_74.jsonl",
"fusion": "rrf",
"rerank": "none",
"channels": [
"passages"
],
"rank_pool": true,
"rank_pool_ce": false,
"n_queries": 74,
"k": 100,
"ndcg@10": 0.5737527941225129,
"ndcg@20": 0.6352462971440838,
"ndcg@50": 0.7092403328584962,
"ndcg": 0.7183777959320791,
"recall@10": 0.46580029319598837,
"recall@20": 0.6964864396614271,
"recall@50": 0.9062083174986402,
"mrr": 0.836599099099099,
"pool_coverage": 0.8462618325454071,
"p50_retrieve_ms": 58.5245,
"p95_retrieve_ms": 82.5653,
"git_sha": "f42c4a8eaadb3c0cc79f78c34b0e1a70534bee10",
"report_path": "runs/search-esci-74-passages-k100-pool/report.json",
"recorded_at": "2026-08-18T20:33:39.620042+00:00"
},
{
"run_id": "search-esci-74-passages-k50-ce",
"brain": "searchbenchesci74",
"dataset": "search_esci_74.jsonl",
"fusion": "rrf",
"rerank": "harness:cross-encoder",
"channels": [
"passages"
],
"rank_pool": false,
"rank_pool_ce": false,
"n_queries": 74,
"k": 50,
"ndcg@10": 0.4478964444706324,
"ndcg@20": 0.5146313751051845,
"ndcg@50": 0.6060742524325241,
"ndcg": 0.6060742524325241,
"recall@10": 0.331273452580599,
"recall@20": 0.5538599992903688,
"recall@50": 0.833954370548067,
"mrr": 0.7197929141477528,
"pool_coverage": null,
"p50_retrieve_ms": 60.091,
"p95_retrieve_ms": 88.02255,
"git_sha": "f42c4a8eaadb3c0cc79f78c34b0e1a70534bee10",
"report_path": "runs/search-esci-74-passages-k50-ce/report.json",
"recorded_at": "2026-08-18T20:34:39.854467+00:00"
},
{
"run_id": "search-esci-74-passages-k50-ce-l12",
"brain": "searchbenchesci74",
"dataset": "search_esci_74.jsonl",
"fusion": "rrf",
"rerank": "harness:cross-encoder",
"channels": [
"passages"
],
"rank_pool": false,
"rank_pool_ce": false,
"n_queries": 74,
"k": 50,
"ndcg@10": 0.4674585131694284,
"ndcg@20": 0.5084668214514952,
"ndcg@50": 0.6140738833603318,
"ndcg": 0.6140738833603318,
"recall@10": 0.3412172585548375,
"recall@20": 0.5191872363321448,
"recall@50": 0.833954370548067,
"mrr": 0.7063349063349063,
"pool_coverage": null,
"p50_retrieve_ms": 60.091,
"p95_retrieve_ms": 88.02255,
"git_sha": "f42c4a8eaadb3c0cc79f78c34b0e1a70534bee10",
"report_path": "runs/search-esci-74-passages-k50-ce-l12/report.json",
"recorded_at": "2026-08-18T20:35:44.462471+00:00"
},
{
"run_id": "search-esci-74-passages-k50-qrewrite",
"brain": "searchbenchesci74",
"dataset": "search_esci_74_qrewrite.jsonl",
"fusion": "rrf",
"rerank": "none",
"channels": [
"passages"
],
"rank_pool": false,
"rank_pool_ce": false,
"n_queries": 74,
"k": 50,
"ndcg@10": 0.4998984443226263,
"ndcg@20": 0.5522842574567189,
"ndcg@50": 0.6378451547901237,
"ndcg": 0.6378451547901237,
"recall@10": 0.378732513089159,
"recall@20": 0.589865760711773,
"recall@50": 0.833954370548067,
"mrr": 0.7645849420849421,
"pool_coverage": null,
"p50_retrieve_ms": 59.432500000000005,
"p95_retrieve_ms": 82.09889999999999,
"git_sha": "f42c4a8eaadb3c0cc79f78c34b0e1a70534bee10",
"report_path": "runs/search-esci-74-passages-k50-qrewrite/report.json",
"recorded_at": "2026-08-18T21:18:51.308816+00:00"
},
{
"run_id": "search-esci-74-dense-ance-k50",
"brain": "harness-local",
"dataset": "search_esci_74.jsonl",
"fusion": "none",
"rerank": "none",
"channels": [
"harness-dense"
],
"rank_pool": false,
"rank_pool_ce": false,
"n_queries": 74,
"k": 50,
"ndcg@10": 0.3960504272587966,
"ndcg@20": 0.445560860839057,
"ndcg@50": 0.5370854731938006,
"ndcg": 0.5370854731938006,
"recall@10": 0.3015013697338196,
"recall@20": 0.4872087291019431,
"recall@50": 0.7503156876126188,
"mrr": 0.6471417846417846,
"pool_coverage": null,
"p50_retrieve_ms": 74.9370945944778,
"p95_retrieve_ms": 74.9370945944778,
"git_sha": "f42c4a8eaadb3c0cc79f78c34b0e1a70534bee10",
"report_path": "runs/search-esci-74-dense-ance-k50/report.json",
"recorded_at": "2026-08-18T21:22:24.054560+00:00"
},
{
"run_id": "search-esci-74-passages-k50-ce-retrieved",
"brain": "searchbenchesci74",
"dataset": "search_esci_74.jsonl",
"fusion": "rrf",
"rerank": "harness:cross-encoder",
"channels": [
"passages"
],
"rank_pool": false,
"rank_pool_ce": false,
"n_queries": 74,
"k": 50,
"ndcg@10": 0.4648895994453083,
"ndcg@20": 0.5245865140860362,
"ndcg@50": 0.6126753191137789,
"ndcg": 0.6126753191137789,
"recall@10": 0.35140463046557896,
"recall@20": 0.5410788442403298,
"recall@50": 0.833954370548067,
"mrr": 0.7120610588001892,
"pool_coverage": null,
"p50_retrieve_ms": 60.091,
"p95_retrieve_ms": 88.02255,
"git_sha": "f42c4a8eaadb3c0cc79f78c34b0e1a70534bee10",
"report_path": "runs/search-esci-74-passages-k50-ce-retrieved/report.json",
"recorded_at": "2026-08-18T22:38:31.141079+00:00"
},
{
"run_id": "search-esci-74-colbert-k50",
"brain": "harness-local-colbert",
"dataset": "search_esci_74.jsonl",
"fusion": "none",
"rerank": "none",
"channels": [
"harness-colbert"
],
"rank_pool": false,
"rank_pool_ce": false,
"n_queries": 74,
"k": 50,
"ndcg@10": 0.43412701768229023,
"ndcg@20": 0.4815601734708754,
"ndcg@50": 0.5523204205368764,
"ndcg": 0.5523204205368764,
"recall@10": 0.3110168068124024,
"recall@20": 0.5023603484326227,
"recall@50": 0.7138820979266567,
"mrr": 0.6689285195333583,
"pool_coverage": null,
"p50_retrieve_ms": 10063.364791662614,
"p95_retrieve_ms": 10063.364791662614,
"git_sha": "f42c4a8eaadb3c0cc79f78c34b0e1a70534bee10",
"report_path": "runs/search-esci-74-colbert-k50/report.json",
"recorded_at": "2026-08-18T22:51:16.979167+00:00"
},
{
"run_id": "search-esci-74-bge-base-k50",
"brain": "harness-local",
"dataset": "search_esci_74.jsonl",
"fusion": "none",
"rerank": "none",
"channels": [
"harness-dense"
],
"rank_pool": false,
"rank_pool_ce": false,
"n_queries": 74,
"k": 50,
"ndcg@10": 0.44136708015333975,
"ndcg@20": 0.5041842920107446,
"ndcg@50": 0.5892299329783445,
"ndcg": 0.5892299329783445,
"recall@10": 0.32179328339461255,
"recall@20": 0.5251414009467592,
"recall@50": 0.7989247943561789,
"mrr": 0.7471303646913403,
"pool_coverage": null,
"p50_retrieve_ms": 408.0482843509762,
"p95_retrieve_ms": 408.04828435097625,
"git_sha": "f42c4a8eaadb3c0cc79f78c34b0e1a70534bee10",
"report_path": "runs/search-esci-74-bge-base-k50/report.json",
"recorded_at": "2026-08-18T22:52:53.742680+00:00"
},
{
"run_id": "search-esci-74-union-bge-k50",
"brain": "harness-local-union",
"dataset": "search_esci_74.jsonl",
"fusion": "rrf",
"rerank": "none",
"channels": [
"harness-union"
],
"rank_pool": false,
"rank_pool_ce": false,
"n_queries": 74,
"k": 50,
"ndcg@10": 0.47918374395281466,
"ndcg@20": 0.5381296158524957,
"ndcg@50": 0.6141255447421892,
"ndcg": 0.6141255447421892,
"recall@10": 0.36314016383757297,
"recall@20": 0.5774058852814317,
"recall@50": 0.8271070974286027,
"mrr": 0.7653163449499656,
"pool_coverage": null,
"p50_retrieve_ms": 0.0,
"p95_retrieve_ms": 0.0,
"git_sha": "f42c4a8eaadb3c0cc79f78c34b0e1a70534bee10",
"report_path": "runs/search-esci-74-union-bge-k50/report.json",
"recorded_at": "2026-08-19T08:10:06.928384+00:00"
},
{
"run_id": "search-esci-74-exhaustive-ce",
"brain": "harness-local-exhaustive",
"dataset": "search_esci_74.jsonl",
"fusion": "none",
"rerank": "4class-weighted-ce",
"channels": [
"exhaustive-4class"
],
"rank_pool": false,
"rank_pool_ce": false,
"n_queries": 74,
"k": 50,
"ndcg@10": 0.4162185585932262,
"ndcg@20": 0.4392534244105597,
"ndcg@50": 0.5042093175793891,
"ndcg": 0.5042093175793891,
"recall@10": 0.26585896457775815,
"recall@20": 0.4060812399456508,
"recall@50": 0.6029270278490679,
"mrr": 0.6578079129936715,
"pool_coverage": null,
"p50_retrieve_ms": 94253.11211881155,
"p95_retrieve_ms": 94253.11211881155,
"git_sha": "f42c4a8eaadb3c0cc79f78c34b0e1a70534bee10",
"report_path": "runs/search-esci-74-exhaustive-ce/report.json",
"recorded_at": "2026-08-19T10:05:59.463503+00:00"
},
{
"run_id": "search-esci-74-catalog-ce-k50",
"brain": "searchbenchesci74",
"dataset": "search_esci_74.jsonl",
"fusion": "rrf",
"rerank": "plugin:cross-encoder",
"channels": [
"passages"
],
"rank_pool": false,
"rank_pool_ce": false,
"n_queries": 74,
"k": 50,
"ndcg@10": 0.4674585131694284,
"ndcg@20": 0.5084668214514952,
"ndcg@50": 0.6140738833603318,
"ndcg": 0.6140738833603318,
"recall@10": 0.3412172585548375,
"recall@20": 0.5191872363321448,
"recall@50": 0.833954370548067,
"mrr": 0.7063349063349063,
"pool_coverage": null,
"p50_retrieve_ms": 59.308499999999995,
"p95_retrieve_ms": 86.88884999999999,
"git_sha": "f42c4a8eaadb3c0cc79f78c34b0e1a70534bee10",
"report_path": "runs/search-esci-74-catalog-ce-k50/report.json",
"recorded_at": "2026-08-19T10:17:45.568452+00:00"
},
{
"run_id": "search-esci-74-cascade-tail-k50",
"brain": "harness-local-cascade",
"dataset": "search_esci_74.jsonl",
"fusion": "none",
"rerank": "none",
"channels": [
"harness-cascade"
],
"rank_pool": false,
"rank_pool_ce": false,
"n_queries": 74,
"k": 50,
"ndcg@10": 0.4998984443226263,
"ndcg@20": 0.6194622913116794,
"ndcg@50": 0.6686100079902446,
"ndcg": 0.6686100079902446,
"recall@10": 0.378732513089159,
"recall@20": 0.7504072036031975,
"recall@50": 0.8885964311815054,
"mrr": 0.7679931554931555,
"pool_coverage": null,
"p50_retrieve_ms": 0.0,
"p95_retrieve_ms": 0.0,
"git_sha": "f42c4a8eaadb3c0cc79f78c34b0e1a70534bee10",
"report_path": "runs/search-esci-74-cascade-tail-k50/report.json",
"recorded_at": "2026-08-19T10:32:17.738702+00:00"
},
{
"run_id": "search-esci-74-ce-pool-deberta-base",
"brain": "searchbenchesci74",
"dataset": "search_esci_74.jsonl",
"fusion": "none",
"rerank": "4class-weighted-ce",
"channels": [
"rank-pool-4class"
],
"rank_pool": true,
"rank_pool_ce": true,
"n_queries": 74,
"k": 40,
"ndcg@10": 0.6492728519674476,
"ndcg@20": 0.7101117922903459,
"ndcg@50": null,
"ndcg": 0.7772540710194906,
"recall@10": 0.5183145850778,
"recall@20": 0.7604145345598889,
"recall@50": null,
"mrr": 0.8646712599298806,
"pool_coverage": null,
"p50_retrieve_ms": 0.0,
"p95_retrieve_ms": 0.0,
"git_sha": "f42c4a8eaadb3c0cc79f78c34b0e1a70534bee10",
"report_path": "runs/search-esci-74-ce-pool-deberta-base/report.json",
"recorded_at": "2026-08-19T14:00:25.594211+00:00"
},
{
"run_id": "search-esci-74-passages-k50-ce-deberta",
"brain": "searchbenchsmoke",
"dataset": "search_esci_74.jsonl",
"fusion": "rrf",
"rerank": "harness:cross-encoder",
"channels": [
"passages"
],
"rank_pool": false,
"rank_pool_ce": false,
"n_queries": 74,
"k": 50,
"ndcg@10": 0.5101053946025641,
"ndcg@20": 0.5568741362838021,
"ndcg@50": 0.6365973490191127,
"ndcg": 0.6365973490191127,
"recall@10": 0.36281504203381093,
"recall@20": 0.5467544580912761,
"recall@50": 0.833954370548067,
"mrr": 0.7457139784725992,
"pool_coverage": null,
"p50_retrieve_ms": 60.091,
"p95_retrieve_ms": 88.02255,
"git_sha": "f42c4a8eaadb3c0cc79f78c34b0e1a70534bee10",
"report_path": "runs/search-esci-74-passages-k50-ce-deberta/report.json",
"recorded_at": "2026-08-19T14:09:19.540811+00:00"
},
{
"run_id": "search-esci-es-passages-k50",
"brain": "searchbenchescies",
"dataset": "search_esci_es.jsonl",
"fusion": "rrf",
"rerank": "none",
"channels": [
"passages"
],
"rank_pool": false,
"rank_pool_ce": false,
"n_queries": 62,
"k": 50,
"ndcg@10": 0.5774393479403104,
"ndcg@20": 0.6303553323648456,
"ndcg@50": 0.7327853549091549,
"ndcg": 0.7327853549091549,
"recall@10": 0.35328224222955834,
"recall@20": 0.5758587932370467,
"recall@50": 0.9141280526823616,
"mrr": 0.8704493087557603,
"pool_coverage": null,
"p50_retrieve_ms": 84.13,
"p95_retrieve_ms": 96.16295000000001,
"git_sha": "f42c4a8eaadb3c0cc79f78c34b0e1a70534bee10",
"report_path": "runs/search-esci-es-passages-k50/report.json",
"recorded_at": "2026-08-19T16:34:42.965337+00:00"
},
{
"run_id": "search-italian-smoke",
"brain": "searchbenchitsmoke",
"dataset": "search_italian_smoke.jsonl",
"fusion": "rrf",
"rerank": "none",
"channels": [
"passages"
],
"rank_pool": false,
"rank_pool_ce": false,
"n_queries": 3,
"k": 20,
"ndcg@10": 1.0,
"ndcg@20": 1.0,
"ndcg@50": null,
"ndcg": 1.0,
"recall@10": 1.0,
"recall@20": 1.0,
"recall@50": null,
"mrr": 1.0,
"pool_coverage": null,
"p50_retrieve_ms": 25.33,
"p95_retrieve_ms": 29.3206,
"git_sha": "f42c4a8eaadb3c0cc79f78c34b0e1a70534bee10",
"report_path": "runs/search-italian-smoke/report.json",
"recorded_at": "2026-08-19T16:37:37.730780+00:00"
},
{
"run_id": "search-italian-smoke-inflect",
"brain": "searchbenchitsmoke",
"dataset": "search_italian_smoke_inflect.jsonl",
"fusion": "rrf",
"rerank": "none",
"channels": [
"passages"
],
"rank_pool": false,
"rank_pool_ce": false,
"n_queries": 3,
"k": 20,
"ndcg@10": 1.0,
"ndcg@20": 1.0,
"ndcg@50": null,
"ndcg": 1.0,
"recall@10": 1.0,
"recall@20": 1.0,
"recall@50": null,
"mrr": 1.0,
"pool_coverage": null,
"p50_retrieve_ms": 21.494,
"p95_retrieve_ms": 23.211199999999998,
"git_sha": "f42c4a8eaadb3c0cc79f78c34b0e1a70534bee10",
"report_path": "runs/search-italian-smoke-inflect/report.json",
"recorded_at": "2026-08-19T16:37:46.108861+00:00"
},
{
"run_id": "search-italian-smoke-inflect-fts",
"brain": "searchbenchitsmoke",
"dataset": "search_italian_smoke_inflect.jsonl",
"fusion": "rrf",
"rerank": "none",
"channels": [
"passages"
],
"rank_pool": false,
"rank_pool_ce": false,
"n_queries": 3,
"k": 20,
"ndcg@10": 1.0,
"ndcg@20": 1.0,
"ndcg@50": null,
"ndcg": 1.0,
"recall@10": 1.0,
"recall@20": 1.0,
"recall@50": null,
"mrr": 1.0,
"pool_coverage": null,
"p50_retrieve_ms": 33.56,
"p95_retrieve_ms": 36.8936,
"git_sha": "f42c4a8eaadb3c0cc79f78c34b0e1a70534bee10",
"report_path": "runs/search-italian-smoke-inflect-fts/report.json",
"recorded_at": "2026-08-19T16:44:05.699330+00:00"
},
{
"run_id": "search-italian-minilm-pipeline",
"brain": "searchbenchitmmini",
"dataset": "search_italian_smoke.jsonl",
"fusion": "none",
"rerank": "none",
"channels": [
"harness-dense"
],
"rank_pool": false,
"rank_pool_ce": false,
"n_queries": 3,
"k": 10,
"ndcg@10": 1.0,
"ndcg@20": 1.0,
"ndcg@50": null,
"ndcg": 1.0,
"recall@10": 1.0,
"recall@20": null,
"recall@50": null,
"mrr": 1.0,
"pool_coverage": null,
"p50_retrieve_ms": 188.19916666689096,
"p95_retrieve_ms": 188.19916666689096,
"git_sha": "f42c4a8eaadb3c0cc79f78c34b0e1a70534bee10",
"report_path": "runs/search-italian-minilm-pipeline/report.json",
"recorded_at": "2026-08-19T19:27:33.792776+00:00"
},
{
"run_id": "search-esci-es-spell-k50",
"brain": "searchbenchescies",
"dataset": "search_esci_es_spell.jsonl",
"fusion": "rrf",
"rerank": "none",
"channels": [
"passages"
],
"rank_pool": false,
"rank_pool_ce": false,
"n_queries": 62,
"k": 50,
"ndcg@10": 0.5950327313726855,
"ndcg@20": 0.6464666665333806,
"ndcg@50": 0.7424579237775919,
"ndcg": 0.7424579237775919,
"recall@10": 0.36220476611566177,
"recall@20": 0.5832920760052671,
"recall@50": 0.9143064382925891,
"mrr": 0.8865783410138248,
"pool_coverage": null,
"p50_retrieve_ms": 92.655,
"p95_retrieve_ms": 107.2923,
"git_sha": "f42c4a8eaadb3c0cc79f78c34b0e1a70534bee10",
"report_path": "runs/search-esci-es-spell-k50/report.json",
"recorded_at": "2026-08-19T19:28:14.068481+00:00"
},
{
"run_id": "search-italian-smoke-extras-locale",
"brain": "searchbenchitsmoke",
"dataset": "search_italian_smoke.jsonl",
"fusion": "rrf",
"rerank": "none",
"channels": [
"passages"
],
"rank_pool": false,
"rank_pool_ce": false,
"n_queries": 3,
"k": 10,
"ndcg@10": 1.0,
"ndcg@20": 1.0,
"ndcg@50": null,
"ndcg": 1.0,
"recall@10": 1.0,
"recall@20": null,
"recall@50": null,
"mrr": 1.0,
"pool_coverage": null,
"p50_retrieve_ms": 17.126,
"p95_retrieve_ms": 23.894000000000002,
"git_sha": "f42c4a8eaadb3c0cc79f78c34b0e1a70534bee10",
"report_path": "runs/search-italian-smoke-extras-locale/report.json",
"recorded_at": "2026-08-19T19:30:47.530480+00:00"
},
{
"run_id": "search-italian-smoke-extras-nope",
"brain": "searchbenchitsmoke",
"dataset": "search_italian_smoke.jsonl",
"fusion": "rrf",
"rerank": "none",
"channels": [
"passages"
],
"rank_pool": false,
"rank_pool_ce": false,
"n_queries": 3,
"k": 10,
"ndcg@10": 0.0,
"ndcg@20": 0.0,
"ndcg@50": null,
"ndcg": 0.0,
"recall@10": 0.0,
"recall@20": null,
"recall@50": null,
"mrr": 0.0,
"pool_coverage": null,
"p50_retrieve_ms": 22.721,
"p95_retrieve_ms": 24.2753,
"git_sha": "f42c4a8eaadb3c0cc79f78c34b0e1a70534bee10",
"report_path": "runs/search-italian-smoke-extras-nope/report.json",
"recorded_at": "2026-08-19T19:29:31.676418+00:00"
},
{
"run_id": "search-esci-74-passages-k50-extras-sanity",
"brain": "searchbenchesci74",
"dataset": "search_esci_74.jsonl",
"fusion": "rrf",
"rerank": "none",
"channels": [
"passages"
],
"rank_pool": false,
"rank_pool_ce": false,
"n_queries": 74,
"k": 50,
"ndcg@10": 0.4998984443226263,
"ndcg@20": 0.5522842574567189,
"ndcg@50": 0.6378451547901237,
"ndcg": 0.6378451547901237,
"recall@10": 0.378732513089159,
"recall@20": 0.589865760711773,
"recall@50": 0.833954370548067,
"mrr": 0.7645849420849421,
"pool_coverage": null,
"p50_retrieve_ms": 63.1325,
"p95_retrieve_ms": 88.86579999999998,
"git_sha": "f42c4a8eaadb3c0cc79f78c34b0e1a70534bee10",
"report_path": "runs/search-esci-74-passages-k50-extras-sanity/report.json",
"recorded_at": "2026-08-19T19:35:44.545077+00:00"
},
{
"run_id": "search-italian-smoke-inflect-fts-or",
"brain": "searchbenchitsmoke",
"dataset": "search_italian_smoke_inflect.jsonl",
"fusion": "rrf",
"rerank": "none",
"channels": [
"passages"
],
"rank_pool": false,
"rank_pool_ce": false,
"n_queries": 3,
"k": 20,
"ndcg@10": 1.0,
"ndcg@20": 1.0,
"ndcg@50": null,
"ndcg": 1.0,
"recall@10": 1.0,
"recall@20": 1.0,
"recall@50": null,
"mrr": 1.0,
"pool_coverage": null,
"p50_retrieve_ms": 26.364,
"p95_retrieve_ms": 42.744899999999994,
"git_sha": "f42c4a8eaadb3c0cc79f78c34b0e1a70534bee10",
"report_path": "runs/search-italian-smoke-inflect-fts-or/report.json",
"recorded_at": "2026-08-19T20:16:24.241357+00:00"
},
{
"run_id": "search-esci-74-cc-a03",
"brain": "searchbenchesci74",
"dataset": "search_esci_74.jsonl",
"fusion": "cc",
"rerank": "none",
"channels": [
"passages"
],
"rank_pool": false,
"rank_pool_ce": false,
"n_queries": 74,
"k": 50,
"ndcg@10": 0.48296913346675313,
"ndcg@20": 0.5353434819101804,
"ndcg@50": 0.6235227747628354,
"ndcg": 0.6235227747628354,
"recall@10": 0.37594464118364007,
"recall@20": 0.5859902261910772,
"recall@50": 0.8324528690465655,
"mrr": 0.7492780202562811,
"pool_coverage": null,
"p50_retrieve_ms": 62.8885,
"p95_retrieve_ms": 89.98204999999999,
"git_sha": "f42c4a8eaadb3c0cc79f78c34b0e1a70534bee10",
"report_path": "runs/search-esci-74-cc-a03/report.json",
"recorded_at": "2026-08-19T20:18:18.361436+00:00"
},
{
"run_id": "search-esci-74-cc-a05",
"brain": "searchbenchesci74",
"dataset": "search_esci_74.jsonl",
"fusion": "cc",
"rerank": "none",
"channels": [
"passages"
],
"rank_pool": false,
"rank_pool_ce": false,
"n_queries": 74,
"k": 50,
"ndcg@10": 0.4861505982058926,
"ndcg@20": 0.5412686017716648,
"ndcg@50": 0.62608241624179,
"ndcg": 0.62608241624179,
"recall@10": 0.37673766109430706,
"recall@20": 0.5919104321112831,
"recall@50": 0.8324528690465655,
"mrr": 0.7604021879021879,
"pool_coverage": null,
"p50_retrieve_ms": 61.432,
"p95_retrieve_ms": 92.56864999999999,
"git_sha": "f42c4a8eaadb3c0cc79f78c34b0e1a70534bee10",
"report_path": "runs/search-esci-74-cc-a05/report.json",
"recorded_at": "2026-08-19T20:18:41.196658+00:00"
},
{
"run_id": "search-esci-74-cc-a07",
"brain": "searchbenchesci74",
"dataset": "search_esci_74.jsonl",
"fusion": "cc",
"rerank": "none",
"channels": [
"passages"
],
"rank_pool": false,
"rank_pool_ce": false,
"n_queries": 74,
"k": 50,
"ndcg@10": 0.4925393217389386,
"ndcg@20": 0.5491258060245199,
"ndcg@50": 0.6321844176410597,
"ndcg": 0.6321844176410597,
"recall@10": 0.3780890124456584,
"recall@20": 0.5964401922862047,
"recall@50": 0.833954370548067,
"mrr": 0.7630083655083656,
"pool_coverage": null,
"p50_retrieve_ms": 60.5155,
"p95_retrieve_ms": 89.04719999999998,
"git_sha": "f42c4a8eaadb3c0cc79f78c34b0e1a70534bee10",
"report_path": "runs/search-esci-74-cc-a07/report.json",
"recorded_at": "2026-08-19T20:19:05.271923+00:00"
},
{
"run_id": "search-esci-74-literal-fill-k50",
"brain": "searchbenchesci74",
"dataset": "search_esci_74.jsonl",
"fusion": "rrf",
"rerank": "none",
"channels": [
"passages"
],
"rank_pool": false,
"rank_pool_ce": false,
"n_queries": 74,
"k": 50,
"ndcg@10": 0.4998984443226263,
"ndcg@20": 0.45027838324634667,
"ndcg@50": 0.5436008239474691,
"ndcg": 0.5436008239474691,
"recall@10": 0.378732513089159,
"recall@20": 0.3929861145019264,
"recall@50": 0.6549969089908488,
"mrr": 0.7639237451737452,
"pool_coverage": null,
"p50_retrieve_ms": 49.019999999999996,
"p95_retrieve_ms": 74.27314999999999,
"git_sha": "f42c4a8eaadb3c0cc79f78c34b0e1a70534bee10",
"report_path": "runs/search-esci-74-literal-fill-k50/report.json",
"recorded_at": "2026-08-19T20:20:15.131585+00:00"
},
{
"run_id": "search-esci-74-ltr-head-k50",
"brain": "harness-local-ltr",
"dataset": "search_esci_74.jsonl",
"fusion": "none",
"rerank": "ltr-pairwise",
"channels": [
"harness-ltr-head"
],
"rank_pool": false,
"rank_pool_ce": false,
"n_queries": 74,
"k": 50,
"ndcg@10": 0.5146015046757452,
"ndcg@20": 0.5567887112159354,
"ndcg@50": 0.6406685515140392,
"ndcg": 0.6406685515140392,
"recall@10": 0.384165323643092,
"recall@20": 0.5830174861607277,
"recall@50": 0.833954370548067,
"mrr": 0.7525579150579151,
"pool_coverage": null,
"p50_retrieve_ms": 0.0,
"p95_retrieve_ms": 0.0,
"git_sha": "f42c4a8eaadb3c0cc79f78c34b0e1a70534bee10",
"report_path": "runs/search-esci-74-ltr-head-k50/report.json",
"recorded_at": "2026-08-19T21:09:47.411344+00:00"
},
{
"run_id": "search-esci-74-ltr-cefeat-k50",
"brain": "harness-local-ltr",
"dataset": "search_esci_74.jsonl",
"fusion": "none",
"rerank": "ltr-pairwise",
"channels": [
"harness-ltr-head"
],
"rank_pool": false,
"rank_pool_ce": false,
"n_queries": 74,
"k": 50,
"ndcg@10": 0.5242780297400812,
"ndcg@20": 0.5655195290162985,
"ndcg@50": 0.6456032400547804,
"ndcg": 0.6456032400547804,
"recall@10": 0.38343544674019964,
"recall@20": 0.5687827599719494,
"recall@50": 0.833954370548067,
"mrr": 0.7792567567567568,
"pool_coverage": null,
"p50_retrieve_ms": 0.0,
"p95_retrieve_ms": 0.0,
"git_sha": "f42c4a8eaadb3c0cc79f78c34b0e1a70534bee10",
"report_path": "runs/search-esci-74-ltr-cefeat-k50/report.json",
"recorded_at": "2026-08-20T07:14:39.761450+00:00"
},
{
"run_id": "search-esci-74-ltr-deberta-k50",
"brain": "harness-local-ltr",
"dataset": "search_esci_74.jsonl",
"fusion": "none",
"rerank": "ltr-pairwise",
"channels": [
"harness-ltr-head"
],
"rank_pool": false,
"rank_pool_ce": false,
"n_queries": 74,
"k": 50,
"ndcg@10": 0.5420964444237363,
"ndcg@20": 0.580013195834899,
"ndcg@50": 0.6509103805030941,
"ndcg": 0.6509103805030941,
"recall@10": 0.3872531182148088,
"recall@20": 0.5863055009035371,
"recall@50": 0.833954370548067,
"mrr": 0.7744787644787645,
"pool_coverage": null,
"p50_retrieve_ms": 0.0,
"p95_retrieve_ms": 0.0,
"git_sha": "f42c4a8eaadb3c0cc79f78c34b0e1a70534bee10",
"report_path": "runs/search-esci-74-ltr-deberta-k50/report.json",
"recorded_at": "2026-08-20T07:43:07.017767+00:00"
},
{
"run_id": "search-esci-74-ltr-deberta-lgbm-k50",
"brain": "harness-local-ltr",
"dataset": "search_esci_74.jsonl",
"fusion": "none",
"rerank": "ltr-lightgbm",
"channels": [
"harness-ltr-head"
],
"rank_pool": false,
"rank_pool_ce": false,
"n_queries": 74,
"k": 50,
"ndcg@10": 0.5159432058815566,
"ndcg@20": 0.5658120484311862,
"ndcg@50": 0.6415336394479464,
"ndcg": 0.6415336394479464,
"recall@10": 0.37878019077485103,
"recall@20": 0.5918308018978374,
"recall@50": 0.833954370548067,
"mrr": 0.7098312215959275,
"pool_coverage": null,
"p50_retrieve_ms": 0.0,
"p95_retrieve_ms": 0.0,
"git_sha": "f42c4a8eaadb3c0cc79f78c34b0e1a70534bee10",
"report_path": "runs/search-esci-74-ltr-deberta-lgbm-k50/report.json",
"recorded_at": "2026-08-20T07:48:40.794647+00:00"
},
{
"run_id": "search-esci-ltr200-passages-k50",
"brain": "searchbenchesciltr2",
"dataset": "search_esci_ltr200.jsonl",
"fusion": "rrf",
"rerank": "none",
"channels": [
"passages"
],
"rank_pool": false,
"rank_pool_ce": false,
"n_queries": 170,
"k": 50,
"ndcg@10": 0.38688497520690734,
"ndcg@20": 0.4428785087667844,
"ndcg@50": 0.5191838961643084,
"ndcg": 0.5191838961643084,
"recall@10": 0.3668275379939918,
"recall@20": 0.5506847180507305,
"recall@50": 0.7435914662537475,
"mrr": 0.6599425680048517,
"pool_coverage": null,
"p50_retrieve_ms": 76.07849999999999,
"p95_retrieve_ms": 152.37659999999983,
"git_sha": "f42c4a8eaadb3c0cc79f78c34b0e1a70534bee10",
"report_path": "runs/search-esci-ltr200-passages-k50/report.json",
"recorded_at": "2026-08-20T10:45:30.571169+00:00"
},
{
"run_id": "search-esci-74-ltr-deberta-train200",
"brain": "harness-local-ltr",
"dataset": "search_esci_74.jsonl",
"fusion": "none",
"rerank": "ltr-pairwise",
"channels": [
"harness-ltr-head"
],
"rank_pool": false,
"rank_pool_ce": false,
"n_queries": 74,
"k": 50,
"ndcg@10": 0.5436096369070966,
"ndcg@20": 0.5825504636416924,
"ndcg@50": 0.6525856561608427,
"ndcg": 0.6525856561608427,
"recall@10": 0.3911684808472767,
"recall@20": 0.5850220289937974,
"recall@50": 0.833954370548067,
"mrr": 0.7621138996138996,
"pool_coverage": null,
"p50_retrieve_ms": 0.0,
"p95_retrieve_ms": 0.0,
"git_sha": "f42c4a8eaadb3c0cc79f78c34b0e1a70534bee10",
"report_path": "runs/search-esci-74-ltr-deberta-train200/report.json",
"recorded_at": "2026-08-20T10:57:34.082897+00:00"
},
{
"run_id": "search-esci-74-ltr-deberta-lgbm-train200",
"brain": "harness-local-ltr",
"dataset": "search_esci_74.jsonl",
"fusion": "none",
"rerank": "ltr-lightgbm",
"channels": [
"harness-ltr-head"
],
"rank_pool": false,
"rank_pool_ce": false,
"n_queries": 74,
"k": 50,
"ndcg@10": 0.5333470812074246,
"ndcg@20": 0.5739148370361776,
"ndcg@50": 0.6521013344978932,
"ndcg": 0.6521013344978932,
"recall@10": 0.38938710235182716,
"recall@20": 0.5909398273461974,
"recall@50": 0.833954370548067,
"mrr": 0.7517797970384177,
"pool_coverage": null,
"p50_retrieve_ms": 0.0,
"p95_retrieve_ms": 0.0,
"git_sha": "f42c4a8eaadb3c0cc79f78c34b0e1a70534bee10",
"report_path": "runs/search-esci-74-ltr-deberta-lgbm-train200/report.json",
"recorded_at": "2026-08-20T11:33:18.933082+00:00"
},
{
"run_id": "search-esci-74-ltr-deberta-hybrid170",
"brain": "harness-local-ltr",
"dataset": "search_esci_74.jsonl",
"fusion": "none",
"rerank": "ltr-pairwise",
"channels": [
"harness-ltr-head"
],
"rank_pool": false,
"rank_pool_ce": false,
"n_queries": 74,
"k": 50,
"ndcg@10": 0.5304449022081842,
"ndcg@20": 0.5753013243082012,
"ndcg@50": 0.6452218148321592,
"ndcg": 0.6452218148321592,
"recall@10": 0.38518108116293304,
"recall@20": 0.5882930387092911,
"recall@50": 0.833954370548067,
"mrr": 0.7646718146718147,
"pool_coverage": null,
"p50_retrieve_ms": 0.0,
"p95_retrieve_ms": 0.0,
"git_sha": "f42c4a8eaadb3c0cc79f78c34b0e1a70534bee10",
"report_path": "runs/search-esci-74-ltr-deberta-hybrid170/report.json",
"recorded_at": "2026-08-20T11:57:34.067863+00:00"
},
{
"run_id": "search-wands-passages-k50",
"brain": "searchbenchwands",
"dataset": "search_wands.jsonl",
"fusion": "rrf",
"rerank": "none",
"channels": [
"passages"
],
"rank_pool": false,
"rank_pool_ce": false,
"n_queries": 66,
"k": 50,
"ndcg@10": 0.823176135584089,
"ndcg@20": 0.8120944688355884,
"ndcg@50": 0.8223686140560469,
"ndcg": 0.8223686140560469,
"recall@10": 0.26880378393536286,
"recall@20": 0.46518737110842373,
"recall@50": 0.8365552576078893,
"mrr": 0.9245931537598204,
"pool_coverage": null,
"p50_retrieve_ms": 86.595,
"p95_retrieve_ms": 96.93674999999999,
"git_sha": "f42c4a8eaadb3c0cc79f78c34b0e1a70534bee10",
"report_path": "runs/search-wands-passages-k50/report.json",
"recorded_at": "2026-08-20T13:17:40.353988+00:00"
},
{
"run_id": "search-wandsgraph-communities-k50",
"brain": "searchbenchwandsgraph",
"dataset": "search_wands.jsonl",
"fusion": "rrf",
"rerank": "none",
"channels": [
"communities"
],
"rank_pool": false,
"rank_pool_ce": false,
"n_queries": 66,
"k": 50,
"ndcg@10": 0.4094084735740818,
"ndcg@20": 0.36455789387786824,
"ndcg@50": 0.325224291157982,
"ndcg": 0.4570937299953597,
"recall@10": 0.14824773180036338,
"recall@20": 0.20624528979792137,
"recall@50": 0.28701452056715215,
"mrr": 0.6737914862914863,
"pool_coverage": null,
"p50_retrieve_ms": 5609.237499999999,
"p95_retrieve_ms": 11856.0285,
"git_sha": "f42c4a8eaadb3c0cc79f78c34b0e1a70534bee10",
"report_path": "runs/search-wandsgraph-communities-k50/report.json",
"recorded_at": "2026-08-21T03:12:00.249304+00:00",
"protocol": "architecture demo: isolated catalog graph on searchbenchwandsgraph; not a quality-default claim vs frozen WANDS passages 0.823 or ESCI 0.500",
"claim": "architecture-demo"
},
{
"run_id": "search-wandsgraph-passages-k50",
"brain": "searchbenchwandsgraph",
"dataset": "search_wands.jsonl",
"fusion": "rrf",
"rerank": "none",
"channels": [
"passages"
],
"rank_pool": false,
"rank_pool_ce": false,
"n_queries": 66,
"k": 50,
"ndcg@10": 0.8230998576010691,
"ndcg@20": 0.812056754200934,
"ndcg@50": 0.8223486981324697,
"ndcg": 0.8223486981324697,
"recall@10": 0.26880378393536286,
"recall@20": 0.46518737110842373,
"recall@50": 0.8365552576078893,
"mrr": 0.9245931537598204,
"pool_coverage": null,
"p50_retrieve_ms": 98.604,
"p95_retrieve_ms": 115.12925,
"git_sha": "f42c4a8eaadb3c0cc79f78c34b0e1a70534bee10",
"report_path": "runs/search-wandsgraph-passages-k50/report.json",
"recorded_at": "2026-08-21T03:12:36.455586+00:00",
"protocol": "architecture demo: isolated catalog graph on searchbenchwandsgraph; not a quality-default claim vs frozen WANDS passages 0.823 or ESCI 0.500",
"claim": "architecture-demo"
},
{
"run_id": "search-wandsgraph-passages-nodeid-k50",
"brain": "searchbenchwandsgraph",
"dataset": "search_wands.jsonl",
"fusion": "rrf",
"rerank": "none",
"channels": [
"passages"
],
"rank_pool": false,
"rank_pool_ce": false,
"n_queries": 66,
"k": 50,
"ndcg@10": 0.8230998576010691,
"ndcg@20": 0.812056754200934,
"ndcg@50": 0.8223486981324697,
"ndcg": 0.8223486981324697,
"recall@10": 0.26880378393536286,
"recall@20": 0.46518737110842373,
"recall@50": 0.8365552576078893,
"mrr": 0.9245931537598204,
"pool_coverage": null,
"p50_retrieve_ms": 84.0175,
"p95_retrieve_ms": 105.24199999999999,
"git_sha": "f42c4a8eaadb3c0cc79f78c34b0e1a70534bee10",
"report_path": "runs/search-wandsgraph-passages-nodeid-k50/report.json",
"recorded_at": "2026-08-21T07:53:39.311033+00:00",
"protocol": "architecture demo: isolated catalog graph on searchbenchwandsgraph; not a quality-default claim vs frozen WANDS passages 0.823 or ESCI 0.500",
"claim": "architecture-demo"
}
]
}
}
}