diff --git "a/data.json" "b/data.json" new file mode 100644--- /dev/null +++ "b/data.json" @@ -0,0 +1,5424 @@ +{ + "generated_at": "2026-06-29T19:27:57.081535+00:00", + "date": "2026-06-29", + "benchmarks": [ + "dataset-neonai-qa", + "dataset-nucleotidings-qa", + "dataset-logistics-qa", + "dataset-landuse-wa-qa", + "dataset-shakespeare-qa", + "dataset-futurewise-qa", + "dataset-belebele-qa" + ], + "candidates": { + "retrieval-only::bm25-only::none": { + "id": "retrieval-only::bm25-only::none", + "model": "retrieval-only", + "model_adapter": "brainforge", + "model_repo": "retrieval-only", + "retriever": "bm25-only", + "retriever_adapter": "search_pipeline", + "reranker": "none", + "reranker_model": null, + "is_neon": false, + "is_retrieval_only": true + }, + "retrieval-only::bm25-only::minilm": { + "id": "retrieval-only::bm25-only::minilm", + "model": "retrieval-only", + "model_adapter": "brainforge", + "model_repo": "retrieval-only", + "retriever": "bm25-only", + "retriever_adapter": "search_pipeline", + "reranker": "minilm", + "reranker_model": "cross-encoder/ms-marco-MiniLM-L-6-v2", + "is_neon": false, + "is_retrieval_only": true + }, + "retrieval-only::bm25-only::mxbai": { + "id": "retrieval-only::bm25-only::mxbai", + "model": "retrieval-only", + "model_adapter": "brainforge", + "model_repo": "retrieval-only", + "retriever": "bm25-only", + "retriever_adapter": "search_pipeline", + "reranker": "mxbai", + "reranker_model": "mixedbread-ai/mxbai-rerank-base-v1", + "is_neon": false, + "is_retrieval_only": true + }, + "retrieval-only::embedding-bge-m3::none": { + "id": "retrieval-only::embedding-bge-m3::none", + "model": "retrieval-only", + "model_adapter": "brainforge", + "model_repo": "retrieval-only", + "retriever": "embedding-bge-m3", + "retriever_adapter": "search_pipeline", + "reranker": "none", + "reranker_model": null, + "is_neon": false, + "is_retrieval_only": true + }, + "retrieval-only::embedding-bge-m3::minilm": { + "id": "retrieval-only::embedding-bge-m3::minilm", + "model": "retrieval-only", + "model_adapter": "brainforge", + "model_repo": "retrieval-only", + "retriever": "embedding-bge-m3", + "retriever_adapter": "search_pipeline", + "reranker": "minilm", + "reranker_model": "cross-encoder/ms-marco-MiniLM-L-6-v2", + "is_neon": false, + "is_retrieval_only": true + }, + "retrieval-only::embedding-bge-m3::mxbai": { + "id": "retrieval-only::embedding-bge-m3::mxbai", + "model": "retrieval-only", + "model_adapter": "brainforge", + "model_repo": "retrieval-only", + "retriever": "embedding-bge-m3", + "retriever_adapter": "search_pipeline", + "reranker": "mxbai", + "reranker_model": "mixedbread-ai/mxbai-rerank-base-v1", + "is_neon": false, + "is_retrieval_only": true + }, + "retrieval-only::embedding-bge-large::none": { + "id": "retrieval-only::embedding-bge-large::none", + "model": "retrieval-only", + "model_adapter": "brainforge", + "model_repo": "retrieval-only", + "retriever": "embedding-bge-large", + "retriever_adapter": "search_pipeline", + "reranker": "none", + "reranker_model": null, + "is_neon": false, + "is_retrieval_only": true + }, + "retrieval-only::embedding-bge-large::minilm": { + "id": "retrieval-only::embedding-bge-large::minilm", + "model": "retrieval-only", + "model_adapter": "brainforge", + "model_repo": "retrieval-only", + "retriever": "embedding-bge-large", + "retriever_adapter": "search_pipeline", + "reranker": "minilm", + "reranker_model": "cross-encoder/ms-marco-MiniLM-L-6-v2", + "is_neon": false, + "is_retrieval_only": true + }, + "retrieval-only::embedding-bge-large::mxbai": { + "id": "retrieval-only::embedding-bge-large::mxbai", + "model": "retrieval-only", + "model_adapter": "brainforge", + "model_repo": "retrieval-only", + "retriever": "embedding-bge-large", + "retriever_adapter": "search_pipeline", + "reranker": "mxbai", + "reranker_model": "mixedbread-ai/mxbai-rerank-base-v1", + "is_neon": false, + "is_retrieval_only": true + }, + "retrieval-only::hybrid::none": { + "id": "retrieval-only::hybrid::none", + "model": "retrieval-only", + "model_adapter": "brainforge", + "model_repo": "retrieval-only", + "retriever": "hybrid", + "retriever_adapter": "search_pipeline", + "reranker": "none", + "reranker_model": null, + "is_neon": false, + "is_retrieval_only": true + }, + "retrieval-only::hybrid::minilm": { + "id": "retrieval-only::hybrid::minilm", + "model": "retrieval-only", + "model_adapter": "brainforge", + "model_repo": "retrieval-only", + "retriever": "hybrid", + "retriever_adapter": "search_pipeline", + "reranker": "minilm", + "reranker_model": "cross-encoder/ms-marco-MiniLM-L-6-v2", + "is_neon": false, + "is_retrieval_only": true + }, + "retrieval-only::hybrid::mxbai": { + "id": "retrieval-only::hybrid::mxbai", + "model": "retrieval-only", + "model_adapter": "brainforge", + "model_repo": "retrieval-only", + "retriever": "hybrid", + "retriever_adapter": "search_pipeline", + "reranker": "mxbai", + "reranker_model": "mixedbread-ai/mxbai-rerank-base-v1", + "is_neon": false, + "is_retrieval_only": true + }, + "retrieval-only::hybrid-bge-large::none": { + "id": "retrieval-only::hybrid-bge-large::none", + "model": "retrieval-only", + "model_adapter": "brainforge", + "model_repo": "retrieval-only", + "retriever": "hybrid-bge-large", + "retriever_adapter": "search_pipeline", + "reranker": "none", + "reranker_model": null, + "is_neon": false, + "is_retrieval_only": true + }, + "retrieval-only::hybrid-bge-large::minilm": { + "id": "retrieval-only::hybrid-bge-large::minilm", + "model": "retrieval-only", + "model_adapter": "brainforge", + "model_repo": "retrieval-only", + "retriever": "hybrid-bge-large", + "retriever_adapter": "search_pipeline", + "reranker": "minilm", + "reranker_model": "cross-encoder/ms-marco-MiniLM-L-6-v2", + "is_neon": false, + "is_retrieval_only": true + }, + "retrieval-only::hybrid-bge-large::mxbai": { + "id": "retrieval-only::hybrid-bge-large::mxbai", + "model": "retrieval-only", + "model_adapter": "brainforge", + "model_repo": "retrieval-only", + "retriever": "hybrid-bge-large", + "retriever_adapter": "search_pipeline", + "reranker": "mxbai", + "reranker_model": "mixedbread-ai/mxbai-rerank-base-v1", + "is_neon": false, + "is_retrieval_only": true + }, + "retrieval-only::splade-only::none": { + "id": "retrieval-only::splade-only::none", + "model": "retrieval-only", + "model_adapter": "brainforge", + "model_repo": "retrieval-only", + "retriever": "splade-only", + "retriever_adapter": "search_pipeline", + "reranker": "none", + "reranker_model": null, + "is_neon": false, + "is_retrieval_only": true + }, + "retrieval-only::splade-only::minilm": { + "id": "retrieval-only::splade-only::minilm", + "model": "retrieval-only", + "model_adapter": "brainforge", + "model_repo": "retrieval-only", + "retriever": "splade-only", + "retriever_adapter": "search_pipeline", + "reranker": "minilm", + "reranker_model": "cross-encoder/ms-marco-MiniLM-L-6-v2", + "is_neon": false, + "is_retrieval_only": true + }, + "retrieval-only::splade-only::mxbai": { + "id": "retrieval-only::splade-only::mxbai", + "model": "retrieval-only", + "model_adapter": "brainforge", + "model_repo": "retrieval-only", + "retriever": "splade-only", + "retriever_adapter": "search_pipeline", + "reranker": "mxbai", + "reranker_model": "mixedbread-ai/mxbai-rerank-base-v1", + "is_neon": false, + "is_retrieval_only": true + }, + "retrieval-only::hybrid-triple::none": { + "id": "retrieval-only::hybrid-triple::none", + "model": "retrieval-only", + "model_adapter": "brainforge", + "model_repo": "retrieval-only", + "retriever": "hybrid-triple", + "retriever_adapter": "search_pipeline", + "reranker": "none", + "reranker_model": null, + "is_neon": false, + "is_retrieval_only": true + }, + "retrieval-only::hybrid-triple::minilm": { + "id": "retrieval-only::hybrid-triple::minilm", + "model": "retrieval-only", + "model_adapter": "brainforge", + "model_repo": "retrieval-only", + "retriever": "hybrid-triple", + "retriever_adapter": "search_pipeline", + "reranker": "minilm", + "reranker_model": "cross-encoder/ms-marco-MiniLM-L-6-v2", + "is_neon": false, + "is_retrieval_only": true + }, + "retrieval-only::hybrid-triple::mxbai": { + "id": "retrieval-only::hybrid-triple::mxbai", + "model": "retrieval-only", + "model_adapter": "brainforge", + "model_repo": "retrieval-only", + "retriever": "hybrid-triple", + "retriever_adapter": "search_pipeline", + "reranker": "mxbai", + "reranker_model": "mixedbread-ai/mxbai-rerank-base-v1", + "is_neon": false, + "is_retrieval_only": true + }, + "retrieval-only::hybrid-web::none": { + "id": "retrieval-only::hybrid-web::none", + "model": "retrieval-only", + "model_adapter": "brainforge", + "model_repo": "retrieval-only", + "retriever": "hybrid-web", + "retriever_adapter": "search_pipeline", + "reranker": "none", + "reranker_model": null, + "is_neon": false, + "is_retrieval_only": true + }, + "retrieval-only::hybrid-web::minilm": { + "id": "retrieval-only::hybrid-web::minilm", + "model": "retrieval-only", + "model_adapter": "brainforge", + "model_repo": "retrieval-only", + "retriever": "hybrid-web", + "retriever_adapter": "search_pipeline", + "reranker": "minilm", + "reranker_model": "cross-encoder/ms-marco-MiniLM-L-6-v2", + "is_neon": false, + "is_retrieval_only": true + }, + "retrieval-only::hybrid-web::mxbai": { + "id": "retrieval-only::hybrid-web::mxbai", + "model": "retrieval-only", + "model_adapter": "brainforge", + "model_repo": "retrieval-only", + "retriever": "hybrid-web", + "retriever_adapter": "search_pipeline", + "reranker": "mxbai", + "reranker_model": "mixedbread-ai/mxbai-rerank-base-v1", + "is_neon": false, + "is_retrieval_only": true + }, + "devstral-123b::hybrid::none": { + "id": "devstral-123b::hybrid::none", + "model": "devstral-123b", + "model_adapter": "brainforge", + "model_repo": "devstral-2512:123b", + "retriever": "hybrid", + "retriever_adapter": "search_pipeline", + "reranker": "none", + "reranker_model": null, + "is_neon": true, + "is_retrieval_only": false + }, + "ministral-14b-reasoning::hybrid::none": { + "id": "ministral-14b-reasoning::hybrid::none", + "model": "ministral-14b-reasoning", + "model_adapter": "brainforge", + "model_repo": "mistralai/Ministral-3-14B-Reasoning-2512", + "retriever": "hybrid", + "retriever_adapter": "search_pipeline", + "reranker": "none", + "reranker_model": null, + "is_neon": true, + "is_retrieval_only": false + }, + "magistral-small::hybrid::none": { + "id": "magistral-small::hybrid::none", + "model": "magistral-small", + "model_adapter": "brainforge", + "model_repo": "mistralai/Magistral-Small-2509", + "retriever": "hybrid", + "retriever_adapter": "search_pipeline", + "reranker": "none", + "reranker_model": null, + "is_neon": true, + "is_retrieval_only": false + }, + "devstral-small-24b::hybrid::none": { + "id": "devstral-small-24b::hybrid::none", + "model": "devstral-small-24b", + "model_adapter": "brainforge", + "model_repo": "mistralai/Devstral-Small-2-24B-Instruct-2512", + "retriever": "hybrid", + "retriever_adapter": "search_pipeline", + "reranker": "none", + "reranker_model": null, + "is_neon": true, + "is_retrieval_only": false + }, + "mistral-small-3.2-24b::hybrid::none": { + "id": "mistral-small-3.2-24b::hybrid::none", + "model": "mistral-small-3.2-24b", + "model_adapter": "brainforge", + "model_repo": "mistralai/Mistral-Small-3.2-24B-Instruct-2506", + "retriever": "hybrid", + "retriever_adapter": "search_pipeline", + "reranker": "none", + "reranker_model": null, + "is_neon": true, + "is_retrieval_only": false + }, + "granite-4-30b::hybrid::none": { + "id": "granite-4-30b::hybrid::none", + "model": "granite-4-30b", + "model_adapter": "brainforge", + "model_repo": "ibm-granite/granite-4.1-30b", + "retriever": "hybrid", + "retriever_adapter": "search_pipeline", + "reranker": "none", + "reranker_model": null, + "is_neon": true, + "is_retrieval_only": false + } + }, + "aggregates": [ + { + "candidate_id": "retrieval-only::bm25-only::none", + "benchmark": "dataset-neonai-qa", + "model": "retrieval-only", + "retriever": "bm25-only", + "reranker": "none", + "is_neon": false, + "error": null, + "mrr": 0.4084285714285716, + "hit_at_1": 0.33, + "hit_at_5": 0.52, + "ndcg_at_10": 0.46015234807813926, + "p50_retrieval_latency_ms": 0.19860174506902695, + "p95_retrieval_latency_ms": 0.28101541101932526, + "avg_retrieval_latency_ms": 0.21203815005719662, + "index_build_seconds": 1.3586820373311639, + "est_cost_per_query_usd": 0.0, + "generation_acc": null, + "p50_generation_latency_ms": null, + "avg_generation_latency_ms": null, + "generation_tokens_per_sec": null, + "est_generation_cost_per_query_usd": null, + "paraphrase_drop": 0.06159920634920657, + "shift_drop": 0.13370634920634938, + "n_queries": 100 + }, + { + "candidate_id": "retrieval-only::bm25-only::minilm", + "benchmark": "dataset-neonai-qa", + "model": "retrieval-only", + "retriever": "bm25-only", + "reranker": "minilm", + "is_neon": false, + "error": null, + "mrr": 0.48669444444444454, + "hit_at_1": 0.38, + "hit_at_5": 0.61, + "ndcg_at_10": 0.5465171846177451, + "p50_retrieval_latency_ms": 23.795085959136486, + "p95_retrieval_latency_ms": 27.18852460384369, + "avg_retrieval_latency_ms": 53.217622097581625, + "index_build_seconds": 0.002659284509718418, + "est_cost_per_query_usd": 0.0, + "generation_acc": null, + "p50_generation_latency_ms": null, + "avg_generation_latency_ms": null, + "generation_tokens_per_sec": null, + "est_generation_cost_per_query_usd": null, + "paraphrase_drop": 0.03348809523809515, + "shift_drop": 0.26363888888888903, + "n_queries": 100 + }, + { + "candidate_id": "retrieval-only::bm25-only::mxbai", + "benchmark": "dataset-neonai-qa", + "model": "retrieval-only", + "retriever": "bm25-only", + "reranker": "mxbai", + "is_neon": false, + "error": null, + "mrr": 0.38216269841269834, + "hit_at_1": 0.27, + "hit_at_5": 0.57, + "ndcg_at_10": 0.45257484841179746, + "p50_retrieval_latency_ms": 184.44390688091516, + "p95_retrieval_latency_ms": 188.86269722133875, + "avg_retrieval_latency_ms": 206.04804329574108, + "index_build_seconds": 0.003439362160861492, + "est_cost_per_query_usd": 0.0, + "generation_acc": null, + "p50_generation_latency_ms": null, + "avg_generation_latency_ms": null, + "generation_tokens_per_sec": null, + "est_generation_cost_per_query_usd": null, + "paraphrase_drop": 0.06377777777777771, + "shift_drop": 0.17174603174603165, + "n_queries": 100 + }, + { + "candidate_id": "retrieval-only::embedding-bge-m3::none", + "benchmark": "dataset-neonai-qa", + "model": "retrieval-only", + "retriever": "embedding-bge-m3", + "reranker": "none", + "is_neon": false, + "error": null, + "mrr": 0.4358452380952382, + "hit_at_1": 0.3, + "hit_at_5": 0.63, + "ndcg_at_10": 0.49556998931673163, + "p50_retrieval_latency_ms": 5.7725729420781136, + "p95_retrieval_latency_ms": 6.527797318994999, + "avg_retrieval_latency_ms": 70.81005768850446, + "index_build_seconds": 0.0027428651228547096, + "est_cost_per_query_usd": 0.0, + "generation_acc": null, + "p50_generation_latency_ms": null, + "avg_generation_latency_ms": null, + "generation_tokens_per_sec": null, + "est_generation_cost_per_query_usd": null, + "paraphrase_drop": 0.1001428571428572, + "shift_drop": 0.08578571428571435, + "n_queries": 100 + }, + { + "candidate_id": "retrieval-only::embedding-bge-m3::minilm", + "benchmark": "dataset-neonai-qa", + "model": "retrieval-only", + "retriever": "embedding-bge-m3", + "reranker": "minilm", + "is_neon": false, + "error": null, + "mrr": 0.4400634920634922, + "hit_at_1": 0.35, + "hit_at_5": 0.57, + "ndcg_at_10": 0.49663346937895064, + "p50_retrieval_latency_ms": 30.563555657863617, + "p95_retrieval_latency_ms": 36.099014803767204, + "avg_retrieval_latency_ms": 55.99849511869252, + "index_build_seconds": 0.003172708675265312, + "est_cost_per_query_usd": 0.0, + "generation_acc": null, + "p50_generation_latency_ms": null, + "avg_generation_latency_ms": null, + "generation_tokens_per_sec": null, + "est_generation_cost_per_query_usd": null, + "paraphrase_drop": 0.033496031746031774, + "shift_drop": 0.2262539682539684, + "n_queries": 100 + }, + { + "candidate_id": "retrieval-only::embedding-bge-m3::mxbai", + "benchmark": "dataset-neonai-qa", + "model": "retrieval-only", + "retriever": "embedding-bge-m3", + "reranker": "mxbai", + "is_neon": false, + "error": null, + "mrr": 0.4017301587301587, + "hit_at_1": 0.31, + "hit_at_5": 0.52, + "ndcg_at_10": 0.46009413645764136, + "p50_retrieval_latency_ms": 201.40339527279139, + "p95_retrieval_latency_ms": 204.8659548163414, + "avg_retrieval_latency_ms": 231.20932274498045, + "index_build_seconds": 0.0026545068249106407, + "est_cost_per_query_usd": 0.0, + "generation_acc": null, + "p50_generation_latency_ms": null, + "avg_generation_latency_ms": null, + "generation_tokens_per_sec": null, + "est_generation_cost_per_query_usd": null, + "paraphrase_drop": 0.10631746031746037, + "shift_drop": 0.1821468253968254, + "n_queries": 100 + }, + { + "candidate_id": "retrieval-only::embedding-bge-large::none", + "benchmark": "dataset-neonai-qa", + "model": "retrieval-only", + "retriever": "embedding-bge-large", + "reranker": "none", + "is_neon": false, + "error": null, + "mrr": 0.4435873015873017, + "hit_at_1": 0.36, + "hit_at_5": 0.54, + "ndcg_at_10": 0.49865877957298116, + "p50_retrieval_latency_ms": 5.794810131192207, + "p95_retrieval_latency_ms": 6.4484914764761925, + "avg_retrieval_latency_ms": 111.33181081153452, + "index_build_seconds": 0.0026216572150588036, + "est_cost_per_query_usd": 0.0, + "generation_acc": null, + "p50_generation_latency_ms": null, + "avg_generation_latency_ms": null, + "generation_tokens_per_sec": null, + "est_generation_cost_per_query_usd": null, + "paraphrase_drop": 0.08009126984126985, + "shift_drop": 0.16916269841269854, + "n_queries": 100 + }, + { + "candidate_id": "retrieval-only::embedding-bge-large::minilm", + "benchmark": "dataset-neonai-qa", + "model": "retrieval-only", + "retriever": "embedding-bge-large", + "reranker": "minilm", + "is_neon": false, + "error": null, + "mrr": 0.44914285714285734, + "hit_at_1": 0.34, + "hit_at_5": 0.59, + "ndcg_at_10": 0.5199084083920833, + "p50_retrieval_latency_ms": 30.867163091897964, + "p95_retrieval_latency_ms": 34.51138734817505, + "avg_retrieval_latency_ms": 55.47214945778251, + "index_build_seconds": 0.002636570483446121, + "est_cost_per_query_usd": 0.0, + "generation_acc": null, + "p50_generation_latency_ms": null, + "avg_generation_latency_ms": null, + "generation_tokens_per_sec": null, + "est_generation_cost_per_query_usd": null, + "paraphrase_drop": 0.01790873015873029, + "shift_drop": 0.2918809523809526, + "n_queries": 100 + }, + { + "candidate_id": "retrieval-only::embedding-bge-large::mxbai", + "benchmark": "dataset-neonai-qa", + "model": "retrieval-only", + "retriever": "embedding-bge-large", + "reranker": "mxbai", + "is_neon": false, + "error": null, + "mrr": 0.4083293650793652, + "hit_at_1": 0.32, + "hit_at_5": 0.57, + "ndcg_at_10": 0.46550131063059724, + "p50_retrieval_latency_ms": 203.4943252801895, + "p95_retrieval_latency_ms": 206.6653138026595, + "avg_retrieval_latency_ms": 220.18031349405646, + "index_build_seconds": 0.00274475384503603, + "est_cost_per_query_usd": 0.0, + "generation_acc": null, + "p50_generation_latency_ms": null, + "avg_generation_latency_ms": null, + "generation_tokens_per_sec": null, + "est_generation_cost_per_query_usd": null, + "paraphrase_drop": 0.10934920634920647, + "shift_drop": 0.15874603174603186, + "n_queries": 100 + }, + { + "candidate_id": "retrieval-only::hybrid::none", + "benchmark": "dataset-neonai-qa", + "model": "retrieval-only", + "retriever": "hybrid", + "reranker": "none", + "is_neon": false, + "error": null, + "mrr": 0.5988452380952382, + "hit_at_1": 0.54, + "hit_at_5": 0.68, + "ndcg_at_10": 0.6323249003639557, + "p50_retrieval_latency_ms": 6.271526217460632, + "p95_retrieval_latency_ms": 6.964513100683689, + "avg_retrieval_latency_ms": 6.477495562285185, + "index_build_seconds": 0.032615289092063904, + "est_cost_per_query_usd": 0.0, + "generation_acc": null, + "p50_generation_latency_ms": null, + "avg_generation_latency_ms": null, + "generation_tokens_per_sec": null, + "est_generation_cost_per_query_usd": null, + "paraphrase_drop": 0.07990079365079361, + "shift_drop": 0.3071190476190476, + "n_queries": 100 + }, + { + "candidate_id": "retrieval-only::hybrid::minilm", + "benchmark": "dataset-neonai-qa", + "model": "retrieval-only", + "retriever": "hybrid", + "reranker": "minilm", + "is_neon": false, + "error": null, + "mrr": 0.4738650793650794, + "hit_at_1": 0.36, + "hit_at_5": 0.59, + "ndcg_at_10": 0.5388619648224681, + "p50_retrieval_latency_ms": 33.86049438267946, + "p95_retrieval_latency_ms": 36.04037035256624, + "avg_retrieval_latency_ms": 58.50432200357318, + "index_build_seconds": 0.0028517497703433037, + "est_cost_per_query_usd": 0.0, + "generation_acc": null, + "p50_generation_latency_ms": null, + "avg_generation_latency_ms": null, + "generation_tokens_per_sec": null, + "est_generation_cost_per_query_usd": null, + "paraphrase_drop": 0.01648412698412688, + "shift_drop": 0.2568809523809524, + "n_queries": 100 + }, + { + "candidate_id": "retrieval-only::hybrid::mxbai", + "benchmark": "dataset-neonai-qa", + "model": "retrieval-only", + "retriever": "hybrid", + "reranker": "mxbai", + "is_neon": false, + "error": null, + "mrr": 0.42051190476190475, + "hit_at_1": 0.32, + "hit_at_5": 0.54, + "ndcg_at_10": 0.48588144180113924, + "p50_retrieval_latency_ms": 207.603694871068, + "p95_retrieval_latency_ms": 210.07613744586706, + "avg_retrieval_latency_ms": 224.12402764894068, + "index_build_seconds": 0.030656594783067703, + "est_cost_per_query_usd": 0.0, + "generation_acc": null, + "p50_generation_latency_ms": null, + "avg_generation_latency_ms": null, + "generation_tokens_per_sec": null, + "est_generation_cost_per_query_usd": null, + "paraphrase_drop": 0.1067063492063492, + "shift_drop": 0.16828968253968257, + "n_queries": 100 + }, + { + "candidate_id": "retrieval-only::hybrid-bge-large::none", + "benchmark": "dataset-neonai-qa", + "model": "retrieval-only", + "retriever": "hybrid-bge-large", + "reranker": "none", + "is_neon": false, + "error": null, + "mrr": 0.5777896825396825, + "hit_at_1": 0.52, + "hit_at_5": 0.65, + "ndcg_at_10": 0.6136078722712829, + "p50_retrieval_latency_ms": 6.311035715043545, + "p95_retrieval_latency_ms": 7.2184475138783455, + "avg_retrieval_latency_ms": 6.499620405957103, + "index_build_seconds": 0.0035516079515218735, + "est_cost_per_query_usd": 0.0, + "generation_acc": null, + "p50_generation_latency_ms": null, + "avg_generation_latency_ms": null, + "generation_tokens_per_sec": null, + "est_generation_cost_per_query_usd": null, + "paraphrase_drop": 0.07380555555555546, + "shift_drop": 0.2280674603174604, + "n_queries": 100 + }, + { + "candidate_id": "retrieval-only::hybrid-bge-large::minilm", + "benchmark": "dataset-neonai-qa", + "model": "retrieval-only", + "retriever": "hybrid-bge-large", + "reranker": "minilm", + "is_neon": false, + "error": null, + "mrr": 0.48867063492063506, + "hit_at_1": 0.36, + "hit_at_5": 0.65, + "ndcg_at_10": 0.5624087810600242, + "p50_retrieval_latency_ms": 32.56149683147669, + "p95_retrieval_latency_ms": 35.16612760722637, + "avg_retrieval_latency_ms": 56.71209678053856, + "index_build_seconds": 0.0033905887976288795, + "est_cost_per_query_usd": 0.0, + "generation_acc": null, + "p50_generation_latency_ms": null, + "avg_generation_latency_ms": null, + "generation_tokens_per_sec": null, + "est_generation_cost_per_query_usd": null, + "paraphrase_drop": 0.03553571428571445, + "shift_drop": 0.29494047619047636, + "n_queries": 100 + }, + { + "candidate_id": "retrieval-only::hybrid-bge-large::mxbai", + "benchmark": "dataset-neonai-qa", + "model": "retrieval-only", + "retriever": "hybrid-bge-large", + "reranker": "mxbai", + "is_neon": false, + "error": null, + "mrr": 0.41178174603174617, + "hit_at_1": 0.3, + "hit_at_5": 0.53, + "ndcg_at_10": 0.48585014131635285, + "p50_retrieval_latency_ms": 207.76377152651548, + "p95_retrieval_latency_ms": 210.5171848088503, + "avg_retrieval_latency_ms": 224.80354332365096, + "index_build_seconds": 0.003694332204759121, + "est_cost_per_query_usd": 0.0, + "generation_acc": null, + "p50_generation_latency_ms": null, + "avg_generation_latency_ms": null, + "generation_tokens_per_sec": null, + "est_generation_cost_per_query_usd": null, + "paraphrase_drop": 0.10304761904761917, + "shift_drop": 0.16664285714285731, + "n_queries": 100 + }, + { + "candidate_id": "retrieval-only::splade-only::none", + "benchmark": "dataset-neonai-qa", + "model": "retrieval-only", + "retriever": "splade-only", + "reranker": "none", + "is_neon": false, + "error": null, + "mrr": 0.4040317460317462, + "hit_at_1": 0.33, + "hit_at_5": 0.47, + "ndcg_at_10": 0.45812179966388017, + "p50_retrieval_latency_ms": 1.7055394127964973, + "p95_retrieval_latency_ms": 1.9240109249949455, + "avg_retrieval_latency_ms": 43.49541657604277, + "index_build_seconds": 0.0022735456004738808, + "est_cost_per_query_usd": 0.0, + "generation_acc": null, + "p50_generation_latency_ms": null, + "avg_generation_latency_ms": null, + "generation_tokens_per_sec": null, + "est_generation_cost_per_query_usd": null, + "paraphrase_drop": 0.001369047619047714, + "shift_drop": 0.2857976190476192, + "n_queries": 100 + }, + { + "candidate_id": "retrieval-only::splade-only::minilm", + "benchmark": "dataset-neonai-qa", + "model": "retrieval-only", + "retriever": "splade-only", + "reranker": "minilm", + "is_neon": false, + "error": null, + "mrr": 0.45292460317460337, + "hit_at_1": 0.36, + "hit_at_5": 0.58, + "ndcg_at_10": 0.5158098248243791, + "p50_retrieval_latency_ms": 27.02977228909731, + "p95_retrieval_latency_ms": 29.83796689659357, + "avg_retrieval_latency_ms": 70.1149411778897, + "index_build_seconds": 0.0018904181197285652, + "est_cost_per_query_usd": 0.0, + "generation_acc": null, + "p50_generation_latency_ms": null, + "avg_generation_latency_ms": null, + "generation_tokens_per_sec": null, + "est_generation_cost_per_query_usd": null, + "paraphrase_drop": 0.01390476190476192, + "shift_drop": 0.33814285714285736, + "n_queries": 100 + }, + { + "candidate_id": "retrieval-only::splade-only::mxbai", + "benchmark": "dataset-neonai-qa", + "model": "retrieval-only", + "retriever": "splade-only", + "reranker": "mxbai", + "is_neon": false, + "error": null, + "mrr": 0.37042857142857144, + "hit_at_1": 0.29, + "hit_at_5": 0.48, + "ndcg_at_10": 0.41791732493775036, + "p50_retrieval_latency_ms": 202.1782398223877, + "p95_retrieval_latency_ms": 205.3959835320711, + "avg_retrieval_latency_ms": 218.58474116772413, + "index_build_seconds": 0.0017274552956223488, + "est_cost_per_query_usd": 0.0, + "generation_acc": null, + "p50_generation_latency_ms": null, + "avg_generation_latency_ms": null, + "generation_tokens_per_sec": null, + "est_generation_cost_per_query_usd": null, + "paraphrase_drop": 0.0752936507936508, + "shift_drop": 0.1287619047619048, + "n_queries": 100 + }, + { + "candidate_id": "retrieval-only::hybrid-triple::none", + "benchmark": "dataset-neonai-qa", + "model": "retrieval-only", + "retriever": "hybrid-triple", + "reranker": "none", + "is_neon": false, + "error": null, + "mrr": 0.5843452380952382, + "hit_at_1": 0.52, + "hit_at_5": 0.66, + "ndcg_at_10": 0.6189039868158873, + "p50_retrieval_latency_ms": 7.933019660413265, + "p95_retrieval_latency_ms": 8.787033148109913, + "avg_retrieval_latency_ms": 8.227292886003852, + "index_build_seconds": 0.003947743214666843, + "est_cost_per_query_usd": 0.0, + "generation_acc": null, + "p50_generation_latency_ms": null, + "avg_generation_latency_ms": null, + "generation_tokens_per_sec": null, + "est_generation_cost_per_query_usd": null, + "paraphrase_drop": 0.08083333333333331, + "shift_drop": 0.27720238095238103, + "n_queries": 100 + }, + { + "candidate_id": "retrieval-only::hybrid-triple::minilm", + "benchmark": "dataset-neonai-qa", + "model": "retrieval-only", + "retriever": "hybrid-triple", + "reranker": "minilm", + "is_neon": false, + "error": null, + "mrr": 0.4700000000000001, + "hit_at_1": 0.36, + "hit_at_5": 0.63, + "ndcg_at_10": 0.5404693071256521, + "p50_retrieval_latency_ms": 35.64351052045822, + "p95_retrieval_latency_ms": 40.313600562512875, + "avg_retrieval_latency_ms": 59.9335350561887, + "index_build_seconds": 0.0038605360314249992, + "est_cost_per_query_usd": 0.0, + "generation_acc": null, + "p50_generation_latency_ms": null, + "avg_generation_latency_ms": null, + "generation_tokens_per_sec": null, + "est_generation_cost_per_query_usd": null, + "paraphrase_drop": 0.010769841269841318, + "shift_drop": 0.3127777777777779, + "n_queries": 100 + }, + { + "candidate_id": "retrieval-only::hybrid-triple::mxbai", + "benchmark": "dataset-neonai-qa", + "model": "retrieval-only", + "retriever": "hybrid-triple", + "reranker": "mxbai", + "is_neon": false, + "error": null, + "mrr": 0.41017460317460336, + "hit_at_1": 0.31, + "hit_at_5": 0.52, + "ndcg_at_10": 0.47544300858120564, + "p50_retrieval_latency_ms": 208.5275650024414, + "p95_retrieval_latency_ms": 211.5223752334714, + "avg_retrieval_latency_ms": 224.93235109373927, + "index_build_seconds": 0.0035500377416610718, + "est_cost_per_query_usd": 0.0, + "generation_acc": null, + "p50_generation_latency_ms": null, + "avg_generation_latency_ms": null, + "generation_tokens_per_sec": null, + "est_generation_cost_per_query_usd": null, + "paraphrase_drop": 0.09506349206349224, + "shift_drop": 0.17017460317460342, + "n_queries": 100 + }, + { + "candidate_id": "retrieval-only::hybrid-web::none", + "benchmark": "dataset-neonai-qa", + "model": "retrieval-only", + "retriever": "hybrid-web", + "reranker": "none", + "is_neon": false, + "error": null, + "mrr": 0.5988452380952382, + "hit_at_1": 0.54, + "hit_at_5": 0.68, + "ndcg_at_10": 0.6323249003639557, + "p50_retrieval_latency_ms": 82.29487668722868, + "p95_retrieval_latency_ms": 5015.41096996516, + "avg_retrieval_latency_ms": 1194.4180446770042, + "index_build_seconds": 2.039119898341596, + "est_cost_per_query_usd": 0.0, + "generation_acc": null, + "p50_generation_latency_ms": null, + "avg_generation_latency_ms": null, + "generation_tokens_per_sec": null, + "est_generation_cost_per_query_usd": null, + "paraphrase_drop": 0.07990079365079361, + "shift_drop": 0.3071190476190476, + "n_queries": 100 + }, + { + "candidate_id": "retrieval-only::hybrid-web::minilm", + "benchmark": "dataset-neonai-qa", + "model": "retrieval-only", + "retriever": "hybrid-web", + "reranker": "minilm", + "is_neon": false, + "error": null, + "mrr": 0.4738650793650794, + "hit_at_1": 0.36, + "hit_at_5": 0.59, + "ndcg_at_10": 0.5388619648224681, + "p50_retrieval_latency_ms": 5067.3279808834195, + "p95_retrieval_latency_ms": 5079.049815423787, + "avg_retrieval_latency_ms": 5086.534645138308, + "index_build_seconds": 1.7707714615389705, + "est_cost_per_query_usd": 0.0, + "generation_acc": null, + "p50_generation_latency_ms": null, + "avg_generation_latency_ms": null, + "generation_tokens_per_sec": null, + "est_generation_cost_per_query_usd": null, + "paraphrase_drop": 0.01648412698412688, + "shift_drop": 0.2568809523809524, + "n_queries": 100 + }, + { + "candidate_id": "retrieval-only::hybrid-web::mxbai", + "benchmark": "dataset-neonai-qa", + "model": "retrieval-only", + "retriever": "hybrid-web", + "reranker": "mxbai", + "is_neon": false, + "error": null, + "mrr": 0.42051190476190475, + "hit_at_1": 0.32, + "hit_at_5": 0.54, + "ndcg_at_10": 0.48588144180113924, + "p50_retrieval_latency_ms": 5199.731380678713, + "p95_retrieval_latency_ms": 5228.679247200489, + "avg_retrieval_latency_ms": 5222.45318280533, + "index_build_seconds": 1.761662513948977, + "est_cost_per_query_usd": 0.0, + "generation_acc": null, + "p50_generation_latency_ms": null, + "avg_generation_latency_ms": null, + "generation_tokens_per_sec": null, + "est_generation_cost_per_query_usd": null, + "paraphrase_drop": 0.1067063492063492, + "shift_drop": 0.16828968253968257, + "n_queries": 100 + }, + { + "candidate_id": "retrieval-only::bm25-only::none", + "benchmark": "dataset-nucleotidings-qa", + "model": "retrieval-only", + "retriever": "bm25-only", + "reranker": "none", + "is_neon": false, + "error": null, + "mrr": 0.03176190476190476, + "hit_at_1": 0.0, + "hit_at_5": 0.05, + "ndcg_at_10": 0.056450340414280334, + "p50_retrieval_latency_ms": 0.28319843113422394, + "p95_retrieval_latency_ms": 0.7622698321938515, + "avg_retrieval_latency_ms": 0.3548811189830303, + "index_build_seconds": 0.020172995515167713, + "est_cost_per_query_usd": 0.0, + "generation_acc": null, + "p50_generation_latency_ms": null, + "avg_generation_latency_ms": null, + "generation_tokens_per_sec": null, + "est_generation_cost_per_query_usd": null, + "paraphrase_drop": 0.010376984126984123, + "shift_drop": 0.0016031746031746029, + "n_queries": 100 + }, + { + "candidate_id": "retrieval-only::bm25-only::minilm", + "benchmark": "dataset-nucleotidings-qa", + "model": "retrieval-only", + "retriever": "bm25-only", + "reranker": "minilm", + "is_neon": false, + "error": null, + "mrr": 0.05194444444444444, + "hit_at_1": 0.02, + "hit_at_5": 0.1, + "ndcg_at_10": 0.06818196008810176, + "p50_retrieval_latency_ms": 21.595277823507786, + "p95_retrieval_latency_ms": 22.373918443918228, + "avg_retrieval_latency_ms": 47.56713105365634, + "index_build_seconds": 0.018008974380791187, + "est_cost_per_query_usd": 0.0, + "generation_acc": null, + "p50_generation_latency_ms": null, + "avg_generation_latency_ms": null, + "generation_tokens_per_sec": null, + "est_generation_cost_per_query_usd": null, + "paraphrase_drop": 0.009765873015873013, + "shift_drop": 0.05194444444444444, + "n_queries": 100 + }, + { + "candidate_id": "retrieval-only::bm25-only::mxbai", + "benchmark": "dataset-nucleotidings-qa", + "model": "retrieval-only", + "retriever": "bm25-only", + "reranker": "mxbai", + "is_neon": false, + "error": null, + "mrr": 0.03586904761904762, + "hit_at_1": 0.02, + "hit_at_5": 0.06, + "ndcg_at_10": 0.05063670800167056, + "p50_retrieval_latency_ms": 195.6018004566431, + "p95_retrieval_latency_ms": 199.5778325945139, + "avg_retrieval_latency_ms": 211.85935158282518, + "index_build_seconds": 0.020156115293502808, + "est_cost_per_query_usd": 0.0, + "generation_acc": null, + "p50_generation_latency_ms": null, + "avg_generation_latency_ms": null, + "generation_tokens_per_sec": null, + "est_generation_cost_per_query_usd": null, + "paraphrase_drop": -0.01257539682539683, + "shift_drop": 0.03586904761904762, + "n_queries": 100 + }, + { + "candidate_id": "retrieval-only::embedding-bge-m3::none", + "benchmark": "dataset-nucleotidings-qa", + "model": "retrieval-only", + "retriever": "embedding-bge-m3", + "reranker": "none", + "is_neon": false, + "error": null, + "mrr": 0.03541666666666667, + "hit_at_1": 0.01, + "hit_at_5": 0.06, + "ndcg_at_10": 0.061047911233524085, + "p50_retrieval_latency_ms": 5.855381488800049, + "p95_retrieval_latency_ms": 6.634416989982128, + "avg_retrieval_latency_ms": 6.102179624140263, + "index_build_seconds": 0.04834281001240015, + "est_cost_per_query_usd": 0.0, + "generation_acc": null, + "p50_generation_latency_ms": null, + "avg_generation_latency_ms": null, + "generation_tokens_per_sec": null, + "est_generation_cost_per_query_usd": null, + "paraphrase_drop": 0.007460317460317462, + "shift_drop": 0.03541666666666667, + "n_queries": 100 + }, + { + "candidate_id": "retrieval-only::embedding-bge-m3::minilm", + "benchmark": "dataset-nucleotidings-qa", + "model": "retrieval-only", + "retriever": "embedding-bge-m3", + "reranker": "minilm", + "is_neon": false, + "error": null, + "mrr": 0.05586111111111112, + "hit_at_1": 0.01, + "hit_at_5": 0.13, + "ndcg_at_10": 0.08305216356875389, + "p50_retrieval_latency_ms": 28.12165580689907, + "p95_retrieval_latency_ms": 29.309977777302265, + "avg_retrieval_latency_ms": 52.21044613048434, + "index_build_seconds": 0.019170415587723255, + "est_cost_per_query_usd": 0.0, + "generation_acc": null, + "p50_generation_latency_ms": null, + "avg_generation_latency_ms": null, + "generation_tokens_per_sec": null, + "est_generation_cost_per_query_usd": null, + "paraphrase_drop": 0.010932539682539683, + "shift_drop": 0.05586111111111112, + "n_queries": 100 + }, + { + "candidate_id": "retrieval-only::embedding-bge-m3::mxbai", + "benchmark": "dataset-nucleotidings-qa", + "model": "retrieval-only", + "retriever": "embedding-bge-m3", + "reranker": "mxbai", + "is_neon": false, + "error": null, + "mrr": 0.050857142857142865, + "hit_at_1": 0.02, + "hit_at_5": 0.07, + "ndcg_at_10": 0.0759087886743895, + "p50_retrieval_latency_ms": 206.3822029158473, + "p95_retrieval_latency_ms": 210.27577761560678, + "avg_retrieval_latency_ms": 222.049624286592, + "index_build_seconds": 0.05048837885260582, + "est_cost_per_query_usd": 0.0, + "generation_acc": null, + "p50_generation_latency_ms": null, + "avg_generation_latency_ms": null, + "generation_tokens_per_sec": null, + "est_generation_cost_per_query_usd": null, + "paraphrase_drop": 0.0085, + "shift_drop": 0.050857142857142865, + "n_queries": 100 + }, + { + "candidate_id": "retrieval-only::embedding-bge-large::none", + "benchmark": "dataset-nucleotidings-qa", + "model": "retrieval-only", + "retriever": "embedding-bge-large", + "reranker": "none", + "is_neon": false, + "error": null, + "mrr": 0.030083333333333333, + "hit_at_1": 0.0, + "hit_at_5": 0.07, + "ndcg_at_10": 0.04671055516233946, + "p50_retrieval_latency_ms": 5.988357588648796, + "p95_retrieval_latency_ms": 6.613876670598984, + "avg_retrieval_latency_ms": 6.17221912369132, + "index_build_seconds": 0.01803918555378914, + "est_cost_per_query_usd": 0.0, + "generation_acc": null, + "p50_generation_latency_ms": null, + "avg_generation_latency_ms": null, + "generation_tokens_per_sec": null, + "est_generation_cost_per_query_usd": null, + "paraphrase_drop": 0.003416666666666665, + "shift_drop": 0.030083333333333333, + "n_queries": 100 + }, + { + "candidate_id": "retrieval-only::embedding-bge-large::minilm", + "benchmark": "dataset-nucleotidings-qa", + "model": "retrieval-only", + "retriever": "embedding-bge-large", + "reranker": "minilm", + "is_neon": false, + "error": null, + "mrr": 0.043777777777777784, + "hit_at_1": 0.01, + "hit_at_5": 0.1, + "ndcg_at_10": 0.07039101522343577, + "p50_retrieval_latency_ms": 27.764969505369663, + "p95_retrieval_latency_ms": 29.288841411471367, + "avg_retrieval_latency_ms": 52.290981682017446, + "index_build_seconds": 0.019241033121943474, + "est_cost_per_query_usd": 0.0, + "generation_acc": null, + "p50_generation_latency_ms": null, + "avg_generation_latency_ms": null, + "generation_tokens_per_sec": null, + "est_generation_cost_per_query_usd": null, + "paraphrase_drop": 0.007388888888888889, + "shift_drop": 0.043777777777777784, + "n_queries": 100 + }, + { + "candidate_id": "retrieval-only::embedding-bge-large::mxbai", + "benchmark": "dataset-nucleotidings-qa", + "model": "retrieval-only", + "retriever": "embedding-bge-large", + "reranker": "mxbai", + "is_neon": false, + "error": null, + "mrr": 0.046880952380952384, + "hit_at_1": 0.02, + "hit_at_5": 0.06, + "ndcg_at_10": 0.06816413425401462, + "p50_retrieval_latency_ms": 206.22909720987082, + "p95_retrieval_latency_ms": 210.59383265674114, + "avg_retrieval_latency_ms": 224.03612443245947, + "index_build_seconds": 0.0204993961378932, + "est_cost_per_query_usd": 0.0, + "generation_acc": null, + "p50_generation_latency_ms": null, + "avg_generation_latency_ms": null, + "generation_tokens_per_sec": null, + "est_generation_cost_per_query_usd": null, + "paraphrase_drop": 0.0012619047619047696, + "shift_drop": 0.046880952380952384, + "n_queries": 100 + }, + { + "candidate_id": "retrieval-only::hybrid::none", + "benchmark": "dataset-nucleotidings-qa", + "model": "retrieval-only", + "retriever": "hybrid", + "reranker": "none", + "is_neon": false, + "error": null, + "mrr": 0.048261904761904756, + "hit_at_1": 0.01, + "hit_at_5": 0.1, + "ndcg_at_10": 0.06569076601728688, + "p50_retrieval_latency_ms": 6.575613282620907, + "p95_retrieval_latency_ms": 7.236745208501816, + "avg_retrieval_latency_ms": 6.708775740116835, + "index_build_seconds": 0.0318792499601841, + "est_cost_per_query_usd": 0.0, + "generation_acc": null, + "p50_generation_latency_ms": null, + "avg_generation_latency_ms": null, + "generation_tokens_per_sec": null, + "est_generation_cost_per_query_usd": null, + "paraphrase_drop": 0.015111111111111103, + "shift_drop": 0.048261904761904756, + "n_queries": 100 + }, + { + "candidate_id": "retrieval-only::hybrid::minilm", + "benchmark": "dataset-nucleotidings-qa", + "model": "retrieval-only", + "retriever": "hybrid", + "reranker": "minilm", + "is_neon": false, + "error": null, + "mrr": 0.06034523809523809, + "hit_at_1": 0.02, + "hit_at_5": 0.12, + "ndcg_at_10": 0.08610983041193102, + "p50_retrieval_latency_ms": 29.029096476733685, + "p95_retrieval_latency_ms": 30.2345072850585, + "avg_retrieval_latency_ms": 69.65694262646139, + "index_build_seconds": 0.02931369189172983, + "est_cost_per_query_usd": 0.0, + "generation_acc": null, + "p50_generation_latency_ms": null, + "avg_generation_latency_ms": null, + "generation_tokens_per_sec": null, + "est_generation_cost_per_query_usd": null, + "paraphrase_drop": 0.018138888888888885, + "shift_drop": 0.06034523809523809, + "n_queries": 100 + }, + { + "candidate_id": "retrieval-only::hybrid::mxbai", + "benchmark": "dataset-nucleotidings-qa", + "model": "retrieval-only", + "retriever": "hybrid", + "reranker": "mxbai", + "is_neon": false, + "error": null, + "mrr": 0.05573015873015873, + "hit_at_1": 0.03, + "hit_at_5": 0.09, + "ndcg_at_10": 0.07737790184364975, + "p50_retrieval_latency_ms": 206.8657660856843, + "p95_retrieval_latency_ms": 209.80359707027674, + "avg_retrieval_latency_ms": 223.3355513587594, + "index_build_seconds": 0.0300194276496768, + "est_cost_per_query_usd": 0.0, + "generation_acc": null, + "p50_generation_latency_ms": null, + "avg_generation_latency_ms": null, + "generation_tokens_per_sec": null, + "est_generation_cost_per_query_usd": null, + "paraphrase_drop": 0.014634920634920633, + "shift_drop": 0.05573015873015873, + "n_queries": 100 + }, + { + "candidate_id": "retrieval-only::hybrid-bge-large::none", + "benchmark": "dataset-nucleotidings-qa", + "model": "retrieval-only", + "retriever": "hybrid-bge-large", + "reranker": "none", + "is_neon": false, + "error": null, + "mrr": 0.02687301587301587, + "hit_at_1": 0.0, + "hit_at_5": 0.07, + "ndcg_at_10": 0.048346886529284386, + "p50_retrieval_latency_ms": 6.49019330739975, + "p95_retrieval_latency_ms": 7.493586279451847, + "avg_retrieval_latency_ms": 6.714150300249457, + "index_build_seconds": 0.030312607064843178, + "est_cost_per_query_usd": 0.0, + "generation_acc": null, + "p50_generation_latency_ms": null, + "avg_generation_latency_ms": null, + "generation_tokens_per_sec": null, + "est_generation_cost_per_query_usd": null, + "paraphrase_drop": 0.005111111111111108, + "shift_drop": 0.02687301587301587, + "n_queries": 100 + }, + { + "candidate_id": "retrieval-only::hybrid-bge-large::minilm", + "benchmark": "dataset-nucleotidings-qa", + "model": "retrieval-only", + "retriever": "hybrid-bge-large", + "reranker": "minilm", + "is_neon": false, + "error": null, + "mrr": 0.06802380952380953, + "hit_at_1": 0.01, + "hit_at_5": 0.15, + "ndcg_at_10": 0.10615988438420187, + "p50_retrieval_latency_ms": 29.03958596289158, + "p95_retrieval_latency_ms": 30.06952255964279, + "avg_retrieval_latency_ms": 54.366486547514796, + "index_build_seconds": 0.030157318338751793, + "est_cost_per_query_usd": 0.0, + "generation_acc": null, + "p50_generation_latency_ms": null, + "avg_generation_latency_ms": null, + "generation_tokens_per_sec": null, + "est_generation_cost_per_query_usd": null, + "paraphrase_drop": 0.03101190476190476, + "shift_drop": 0.029758503401360548, + "n_queries": 100 + }, + { + "candidate_id": "retrieval-only::hybrid-bge-large::mxbai", + "benchmark": "dataset-nucleotidings-qa", + "model": "retrieval-only", + "retriever": "hybrid-bge-large", + "reranker": "mxbai", + "is_neon": false, + "error": null, + "mrr": 0.05845238095238095, + "hit_at_1": 0.02, + "hit_at_5": 0.1, + "ndcg_at_10": 0.08229580387362238, + "p50_retrieval_latency_ms": 206.1399845406413, + "p95_retrieval_latency_ms": 209.65103805065155, + "avg_retrieval_latency_ms": 223.03275553509593, + "index_build_seconds": 0.030014587566256523, + "est_cost_per_query_usd": 0.0, + "generation_acc": null, + "p50_generation_latency_ms": null, + "avg_generation_latency_ms": null, + "generation_tokens_per_sec": null, + "est_generation_cost_per_query_usd": null, + "paraphrase_drop": 0.01740476190476191, + "shift_drop": 0.01763605442176871, + "n_queries": 100 + }, + { + "candidate_id": "retrieval-only::splade-only::none", + "benchmark": "dataset-nucleotidings-qa", + "model": "retrieval-only", + "retriever": "splade-only", + "reranker": "none", + "is_neon": false, + "error": null, + "mrr": 0.04010714285714286, + "hit_at_1": 0.0, + "hit_at_5": 0.11, + "ndcg_at_10": 0.0639683234201718, + "p50_retrieval_latency_ms": 3.832290880382061, + "p95_retrieval_latency_ms": 4.106180742383003, + "avg_retrieval_latency_ms": 3.891169335693121, + "index_build_seconds": 0.01208497118204832, + "est_cost_per_query_usd": 0.0, + "generation_acc": null, + "p50_generation_latency_ms": null, + "avg_generation_latency_ms": null, + "generation_tokens_per_sec": null, + "est_generation_cost_per_query_usd": null, + "paraphrase_drop": 0.004273809523809534, + "shift_drop": 0.04010714285714286, + "n_queries": 100 + }, + { + "candidate_id": "retrieval-only::splade-only::minilm", + "benchmark": "dataset-nucleotidings-qa", + "model": "retrieval-only", + "retriever": "splade-only", + "reranker": "minilm", + "is_neon": false, + "error": null, + "mrr": 0.05749603174603173, + "hit_at_1": 0.02, + "hit_at_5": 0.1, + "ndcg_at_10": 0.0837599478144725, + "p50_retrieval_latency_ms": 26.7010610550642, + "p95_retrieval_latency_ms": 27.352082543075085, + "avg_retrieval_latency_ms": 53.00288221798837, + "index_build_seconds": 0.010814650915563107, + "est_cost_per_query_usd": 0.0, + "generation_acc": null, + "p50_generation_latency_ms": null, + "avg_generation_latency_ms": null, + "generation_tokens_per_sec": null, + "est_generation_cost_per_query_usd": null, + "paraphrase_drop": 0.019996031746031734, + "shift_drop": 0.05749603174603173, + "n_queries": 100 + }, + { + "candidate_id": "retrieval-only::splade-only::mxbai", + "benchmark": "dataset-nucleotidings-qa", + "model": "retrieval-only", + "retriever": "splade-only", + "reranker": "mxbai", + "is_neon": false, + "error": null, + "mrr": 0.05083333333333334, + "hit_at_1": 0.02, + "hit_at_5": 0.1, + "ndcg_at_10": 0.07383673926531788, + "p50_retrieval_latency_ms": 203.553250990808, + "p95_retrieval_latency_ms": 207.03642163425684, + "avg_retrieval_latency_ms": 221.47191389463842, + "index_build_seconds": 0.03780477773398161, + "est_cost_per_query_usd": 0.0, + "generation_acc": null, + "p50_generation_latency_ms": null, + "avg_generation_latency_ms": null, + "generation_tokens_per_sec": null, + "est_generation_cost_per_query_usd": null, + "paraphrase_drop": 0.014710317460317465, + "shift_drop": 0.05083333333333334, + "n_queries": 100 + }, + { + "candidate_id": "retrieval-only::hybrid-triple::none", + "benchmark": "dataset-nucleotidings-qa", + "model": "retrieval-only", + "retriever": "hybrid-triple", + "reranker": "none", + "is_neon": false, + "error": null, + "mrr": 0.057595238095238095, + "hit_at_1": 0.02, + "hit_at_5": 0.1, + "ndcg_at_10": 0.07731042095533185, + "p50_retrieval_latency_ms": 11.1324368044734, + "p95_retrieval_latency_ms": 12.249251827597618, + "avg_retrieval_latency_ms": 11.381908534094691, + "index_build_seconds": 0.03214253671467304, + "est_cost_per_query_usd": 0.0, + "generation_acc": null, + "p50_generation_latency_ms": null, + "avg_generation_latency_ms": null, + "generation_tokens_per_sec": null, + "est_generation_cost_per_query_usd": null, + "paraphrase_drop": 0.020178571428571428, + "shift_drop": 0.057595238095238095, + "n_queries": 100 + }, + { + "candidate_id": "retrieval-only::hybrid-triple::minilm", + "benchmark": "dataset-nucleotidings-qa", + "model": "retrieval-only", + "retriever": "hybrid-triple", + "reranker": "minilm", + "is_neon": false, + "error": null, + "mrr": 0.0634126984126984, + "hit_at_1": 0.02, + "hit_at_5": 0.11, + "ndcg_at_10": 0.09714608128796602, + "p50_retrieval_latency_ms": 33.852593973279, + "p95_retrieval_latency_ms": 34.9635211750865, + "avg_retrieval_latency_ms": 58.79330011084676, + "index_build_seconds": 0.03294236399233341, + "est_cost_per_query_usd": 0.0, + "generation_acc": null, + "p50_generation_latency_ms": null, + "avg_generation_latency_ms": null, + "generation_tokens_per_sec": null, + "est_generation_cost_per_query_usd": null, + "paraphrase_drop": 0.020718253968253955, + "shift_drop": 0.0634126984126984, + "n_queries": 100 + }, + { + "candidate_id": "retrieval-only::hybrid-triple::mxbai", + "benchmark": "dataset-nucleotidings-qa", + "model": "retrieval-only", + "retriever": "hybrid-triple", + "reranker": "mxbai", + "is_neon": false, + "error": null, + "mrr": 0.05472222222222222, + "hit_at_1": 0.03, + "hit_at_5": 0.08, + "ndcg_at_10": 0.07404850529840498, + "p50_retrieval_latency_ms": 207.9247822985053, + "p95_retrieval_latency_ms": 210.9046708792448, + "avg_retrieval_latency_ms": 225.0969840027392, + "index_build_seconds": 0.03245750721544027, + "est_cost_per_query_usd": 0.0, + "generation_acc": null, + "p50_generation_latency_ms": null, + "avg_generation_latency_ms": null, + "generation_tokens_per_sec": null, + "est_generation_cost_per_query_usd": null, + "paraphrase_drop": 0.00942063492063492, + "shift_drop": 0.05472222222222222, + "n_queries": 100 + }, + { + "candidate_id": "retrieval-only::hybrid-web::none", + "benchmark": "dataset-nucleotidings-qa", + "model": "retrieval-only", + "retriever": "hybrid-web", + "reranker": "none", + "is_neon": false, + "error": null, + "mrr": 0.048261904761904756, + "hit_at_1": 0.01, + "hit_at_5": 0.1, + "ndcg_at_10": 0.06569076601728688, + "p50_retrieval_latency_ms": 5015.710390172899, + "p95_retrieval_latency_ms": 5045.650668442249, + "avg_retrieval_latency_ms": 5023.558369465172, + "index_build_seconds": 29.299102989025414, + "est_cost_per_query_usd": 0.0, + "generation_acc": null, + "p50_generation_latency_ms": null, + "avg_generation_latency_ms": null, + "generation_tokens_per_sec": null, + "est_generation_cost_per_query_usd": null, + "paraphrase_drop": 0.015111111111111103, + "shift_drop": 0.048261904761904756, + "n_queries": 100 + }, + { + "candidate_id": "retrieval-only::hybrid-web::minilm", + "benchmark": "dataset-nucleotidings-qa", + "model": "retrieval-only", + "retriever": "hybrid-web", + "reranker": "minilm", + "is_neon": false, + "error": null, + "mrr": 0.06034523809523809, + "hit_at_1": 0.02, + "hit_at_5": 0.12, + "ndcg_at_10": 0.08610983041193102, + "p50_retrieval_latency_ms": 5062.78524082154, + "p95_retrieval_latency_ms": 5068.56173556298, + "avg_retrieval_latency_ms": 5080.987708000466, + "index_build_seconds": 26.2965495781973, + "est_cost_per_query_usd": 0.0, + "generation_acc": null, + "p50_generation_latency_ms": null, + "avg_generation_latency_ms": null, + "generation_tokens_per_sec": null, + "est_generation_cost_per_query_usd": null, + "paraphrase_drop": 0.018138888888888885, + "shift_drop": 0.06034523809523809, + "n_queries": 100 + }, + { + "candidate_id": "retrieval-only::hybrid-web::mxbai", + "benchmark": "dataset-nucleotidings-qa", + "model": "retrieval-only", + "retriever": "hybrid-web", + "reranker": "mxbai", + "is_neon": false, + "error": null, + "mrr": 0.05573015873015873, + "hit_at_1": 0.03, + "hit_at_5": 0.09, + "ndcg_at_10": 0.07737790184364975, + "p50_retrieval_latency_ms": 5195.400764234364, + "p95_retrieval_latency_ms": 5222.220067866147, + "avg_retrieval_latency_ms": 5219.289752468467, + "index_build_seconds": 26.202580992132425, + "est_cost_per_query_usd": 0.0, + "generation_acc": null, + "p50_generation_latency_ms": null, + "avg_generation_latency_ms": null, + "generation_tokens_per_sec": null, + "est_generation_cost_per_query_usd": null, + "paraphrase_drop": 0.014634920634920633, + "shift_drop": 0.05573015873015873, + "n_queries": 100 + }, + { + "candidate_id": "devstral-123b::hybrid::none", + "benchmark": "dataset-neonai-qa", + "model": "devstral-123b", + "retriever": "hybrid", + "reranker": "none", + "is_neon": true, + "error": null, + "mrr": 0.5988452380952382, + "hit_at_1": 0.54, + "hit_at_5": 0.68, + "ndcg_at_10": 0.6323249003639557, + "p50_retrieval_latency_ms": 7.858959957957268, + "p95_retrieval_latency_ms": 10.774773545563221, + "avg_retrieval_latency_ms": 8.430057065561414, + "index_build_seconds": 0.002399919554591179, + "est_cost_per_query_usd": 0.0011859399999999997, + "generation_acc": 0.92, + "p50_generation_latency_ms": 640.4259856790304, + "avg_generation_latency_ms": 795.348126348108, + "generation_tokens_per_sec": 712.2525639619537, + "est_generation_cost_per_query_usd": 0.0011859399999999997, + "paraphrase_drop": 0.07990079365079361, + "shift_drop": 0.3071190476190476, + "n_queries": 100 + }, + { + "candidate_id": "devstral-123b::hybrid::none", + "benchmark": "dataset-nucleotidings-qa", + "model": "devstral-123b", + "retriever": "hybrid", + "reranker": "none", + "is_neon": true, + "error": null, + "mrr": 0.048261904761904756, + "hit_at_1": 0.01, + "hit_at_5": 0.1, + "ndcg_at_10": 0.06569076601728688, + "p50_retrieval_latency_ms": 8.428102359175682, + "p95_retrieval_latency_ms": 9.656098671257496, + "avg_retrieval_latency_ms": 8.969784323126078, + "index_build_seconds": 0.022853504866361618, + "est_cost_per_query_usd": 0.0013255999999999993, + "generation_acc": 0.66, + "p50_generation_latency_ms": 703.2549865543842, + "avg_generation_latency_ms": 882.6579355821013, + "generation_tokens_per_sec": 719.36760451188, + "est_generation_cost_per_query_usd": 0.0013255999999999993, + "paraphrase_drop": 0.015111111111111103, + "shift_drop": 0.048261904761904756, + "n_queries": 100 + }, + { + "candidate_id": "devstral-123b::hybrid::none", + "benchmark": "dataset-logistics-qa", + "model": "devstral-123b", + "retriever": "hybrid", + "reranker": "none", + "is_neon": true, + "error": null, + "mrr": 1.0, + "hit_at_1": 1.0, + "hit_at_5": 1.0, + "ndcg_at_10": 1.0, + "p50_retrieval_latency_ms": 7.7611226588487625, + "p95_retrieval_latency_ms": 8.55210516601801, + "avg_retrieval_latency_ms": 7.866447418928146, + "index_build_seconds": 0.0368603216484189, + "est_cost_per_query_usd": 0.001025, + "generation_acc": 0.71, + "p50_generation_latency_ms": 617.4995861947536, + "avg_generation_latency_ms": 736.3612976484001, + "generation_tokens_per_sec": 676.4109967916651, + "est_generation_cost_per_query_usd": 0.001025, + "paraphrase_drop": 0.0, + "shift_drop": 0.0, + "n_queries": 100 + }, + { + "candidate_id": "devstral-123b::hybrid::none", + "benchmark": "dataset-landuse-wa-qa", + "model": "devstral-123b", + "retriever": "hybrid", + "reranker": "none", + "is_neon": true, + "error": null, + "mrr": 1.0, + "hit_at_1": 1.0, + "hit_at_5": 1.0, + "ndcg_at_10": 1.0, + "p50_retrieval_latency_ms": 7.857654243707657, + "p95_retrieval_latency_ms": 8.58728401362896, + "avg_retrieval_latency_ms": 8.052368555217981, + "index_build_seconds": 0.007834444753825665, + "est_cost_per_query_usd": 0.0012769, + "generation_acc": 0.89, + "p50_generation_latency_ms": 794.1247960552573, + "avg_generation_latency_ms": 883.8255668338388, + "generation_tokens_per_sec": 705.0630370070047, + "est_generation_cost_per_query_usd": 0.0012769, + "paraphrase_drop": 0.0, + "shift_drop": 1.0, + "n_queries": 100 + }, + { + "candidate_id": "devstral-123b::hybrid::none", + "benchmark": "dataset-shakespeare-qa", + "model": "devstral-123b", + "retriever": "hybrid", + "reranker": "none", + "is_neon": true, + "error": null, + "mrr": 1.0, + "hit_at_1": 1.0, + "hit_at_5": 1.0, + "ndcg_at_10": 1.0, + "p50_retrieval_latency_ms": 8.404421620070934, + "p95_retrieval_latency_ms": 8.764853700995445, + "avg_retrieval_latency_ms": 8.54923996143043, + "index_build_seconds": 0.05274257529526949, + "est_cost_per_query_usd": 0.0005438600000000001, + "generation_acc": 0.71, + "p50_generation_latency_ms": 461.6543473675847, + "avg_generation_latency_ms": 579.4168068934232, + "generation_tokens_per_sec": 556.6365243323613, + "est_generation_cost_per_query_usd": 0.0005438600000000001, + "paraphrase_drop": 0.0, + "shift_drop": 0.0, + "n_queries": 100 + }, + { + "candidate_id": "devstral-123b::hybrid::none", + "benchmark": "dataset-futurewise-qa", + "model": "devstral-123b", + "retriever": "hybrid", + "reranker": "none", + "is_neon": true, + "error": null, + "mrr": 0.09780952380952382, + "hit_at_1": 0.03, + "hit_at_5": 0.19, + "ndcg_at_10": 0.13218233672282742, + "p50_retrieval_latency_ms": 8.942721411585808, + "p95_retrieval_latency_ms": 9.492317214608192, + "avg_retrieval_latency_ms": 8.958483096212149, + "index_build_seconds": 0.07995483838021755, + "est_cost_per_query_usd": 0.0005834799999999999, + "generation_acc": 0.77, + "p50_generation_latency_ms": 445.1367845758796, + "avg_generation_latency_ms": 535.1428753789514, + "generation_tokens_per_sec": 584.1880310153064, + "est_generation_cost_per_query_usd": 0.0005834799999999999, + "paraphrase_drop": 0.03988095238095239, + "shift_drop": 0.04168707482993198, + "n_queries": 100 + }, + { + "candidate_id": "devstral-123b::hybrid::none", + "benchmark": "dataset-belebele-qa", + "model": "devstral-123b", + "retriever": "hybrid", + "reranker": "none", + "is_neon": true, + "error": null, + "mrr": 0.9678571428571429, + "hit_at_1": 0.95, + "hit_at_5": 0.9928571428571429, + "ndcg_at_10": 0.974179086645947, + "p50_retrieval_latency_ms": 8.227682672441006, + "p95_retrieval_latency_ms": 8.662175387144089, + "avg_retrieval_latency_ms": 8.255848175446902, + "index_build_seconds": 0.031107299029827118, + "est_cost_per_query_usd": 0.000441642857142857, + "generation_acc": 0.9, + "p50_generation_latency_ms": 418.36012806743383, + "avg_generation_latency_ms": 428.0198134681476, + "generation_tokens_per_sec": 503.8791067857494, + "est_generation_cost_per_query_usd": 0.000441642857142857, + "paraphrase_drop": 0.06440476190476185, + "shift_drop": 0.0035714285714285587, + "n_queries": 140 + }, + { + "candidate_id": "ministral-14b-reasoning::hybrid::none", + "benchmark": "dataset-neonai-qa", + "model": "ministral-14b-reasoning", + "retriever": "hybrid", + "reranker": "none", + "is_neon": true, + "error": null, + "mrr": 0.5988452380952382, + "hit_at_1": 0.54, + "hit_at_5": 0.68, + "ndcg_at_10": 0.6323249003639557, + "p50_retrieval_latency_ms": 7.1721151471138, + "p95_retrieval_latency_ms": 7.782273925840855, + "avg_retrieval_latency_ms": 7.279454693198204, + "index_build_seconds": 0.03248952701687813, + "est_cost_per_query_usd": 0.0, + "generation_acc": null, + "p50_generation_latency_ms": null, + "avg_generation_latency_ms": null, + "generation_tokens_per_sec": null, + "est_generation_cost_per_query_usd": null, + "paraphrase_drop": 0.07990079365079361, + "shift_drop": 0.3071190476190476, + "n_queries": 100 + }, + { + "candidate_id": "ministral-14b-reasoning::hybrid::none", + "benchmark": "dataset-nucleotidings-qa", + "model": "ministral-14b-reasoning", + "retriever": "hybrid", + "reranker": "none", + "is_neon": true, + "error": null, + "mrr": 0.048261904761904756, + "hit_at_1": 0.01, + "hit_at_5": 0.1, + "ndcg_at_10": 0.06569076601728688, + "p50_retrieval_latency_ms": 7.616497576236725, + "p95_retrieval_latency_ms": 8.379131555557251, + "avg_retrieval_latency_ms": 7.737708697095513, + "index_build_seconds": 0.032947403378784657, + "est_cost_per_query_usd": 0.0, + "generation_acc": null, + "p50_generation_latency_ms": null, + "avg_generation_latency_ms": null, + "generation_tokens_per_sec": null, + "est_generation_cost_per_query_usd": null, + "paraphrase_drop": 0.015111111111111103, + "shift_drop": 0.048261904761904756, + "n_queries": 100 + }, + { + "candidate_id": "ministral-14b-reasoning::hybrid::none", + "benchmark": "dataset-logistics-qa", + "model": "ministral-14b-reasoning", + "retriever": "hybrid", + "reranker": "none", + "is_neon": true, + "error": null, + "mrr": 1.0, + "hit_at_1": 1.0, + "hit_at_5": 1.0, + "ndcg_at_10": 1.0, + "p50_retrieval_latency_ms": 6.865703500807285, + "p95_retrieval_latency_ms": 7.6750898733735085, + "avg_retrieval_latency_ms": 7.112867152318358, + "index_build_seconds": 0.009198973886668682, + "est_cost_per_query_usd": 0.0, + "generation_acc": null, + "p50_generation_latency_ms": null, + "avg_generation_latency_ms": null, + "generation_tokens_per_sec": null, + "est_generation_cost_per_query_usd": null, + "paraphrase_drop": 0.0, + "shift_drop": 0.0, + "n_queries": 100 + }, + { + "candidate_id": "ministral-14b-reasoning::hybrid::none", + "benchmark": "dataset-landuse-wa-qa", + "model": "ministral-14b-reasoning", + "retriever": "hybrid", + "reranker": "none", + "is_neon": true, + "error": null, + "mrr": 1.0, + "hit_at_1": 1.0, + "hit_at_5": 1.0, + "ndcg_at_10": 1.0, + "p50_retrieval_latency_ms": 7.199288345873356, + "p95_retrieval_latency_ms": 7.892884314060211, + "avg_retrieval_latency_ms": 7.272108998149633, + "index_build_seconds": 0.007608828134834766, + "est_cost_per_query_usd": 0.0, + "generation_acc": null, + "p50_generation_latency_ms": null, + "avg_generation_latency_ms": null, + "generation_tokens_per_sec": null, + "est_generation_cost_per_query_usd": null, + "paraphrase_drop": 0.0, + "shift_drop": 1.0, + "n_queries": 100 + }, + { + "candidate_id": "ministral-14b-reasoning::hybrid::none", + "benchmark": "dataset-shakespeare-qa", + "model": "ministral-14b-reasoning", + "retriever": "hybrid", + "reranker": "none", + "is_neon": true, + "error": null, + "mrr": 1.0, + "hit_at_1": 1.0, + "hit_at_5": 1.0, + "ndcg_at_10": 1.0, + "p50_retrieval_latency_ms": 7.541215978562832, + "p95_retrieval_latency_ms": 7.804302498698235, + "avg_retrieval_latency_ms": 7.6186764147132635, + "index_build_seconds": 0.05469726677983999, + "est_cost_per_query_usd": 0.0, + "generation_acc": null, + "p50_generation_latency_ms": null, + "avg_generation_latency_ms": null, + "generation_tokens_per_sec": null, + "est_generation_cost_per_query_usd": null, + "paraphrase_drop": 0.0, + "shift_drop": 0.0, + "n_queries": 100 + }, + { + "candidate_id": "ministral-14b-reasoning::hybrid::none", + "benchmark": "dataset-futurewise-qa", + "model": "ministral-14b-reasoning", + "retriever": "hybrid", + "reranker": "none", + "is_neon": true, + "error": null, + "mrr": 0.09780952380952382, + "hit_at_1": 0.03, + "hit_at_5": 0.19, + "ndcg_at_10": 0.13218233672282742, + "p50_retrieval_latency_ms": 7.985537871718407, + "p95_retrieval_latency_ms": 8.448821492493153, + "avg_retrieval_latency_ms": 8.036215221509337, + "index_build_seconds": 0.08024525735527277, + "est_cost_per_query_usd": 0.0, + "generation_acc": null, + "p50_generation_latency_ms": null, + "avg_generation_latency_ms": null, + "generation_tokens_per_sec": null, + "est_generation_cost_per_query_usd": null, + "paraphrase_drop": 0.03988095238095239, + "shift_drop": 0.04168707482993198, + "n_queries": 100 + }, + { + "candidate_id": "ministral-14b-reasoning::hybrid::none", + "benchmark": "dataset-belebele-qa", + "model": "ministral-14b-reasoning", + "retriever": "hybrid", + "reranker": "none", + "is_neon": true, + "error": null, + "mrr": 0.9678571428571429, + "hit_at_1": 0.95, + "hit_at_5": 0.9928571428571429, + "ndcg_at_10": 0.974179086645947, + "p50_retrieval_latency_ms": 7.509047165513039, + "p95_retrieval_latency_ms": 7.911396212875843, + "avg_retrieval_latency_ms": 7.4360380082258155, + "index_build_seconds": 0.0037398524582386017, + "est_cost_per_query_usd": 0.0, + "generation_acc": null, + "p50_generation_latency_ms": null, + "avg_generation_latency_ms": null, + "generation_tokens_per_sec": null, + "est_generation_cost_per_query_usd": null, + "paraphrase_drop": 0.06440476190476185, + "shift_drop": 0.0035714285714285587, + "n_queries": 140 + }, + { + "candidate_id": "magistral-small::hybrid::none", + "benchmark": "dataset-neonai-qa", + "model": "magistral-small", + "retriever": "hybrid", + "reranker": "none", + "is_neon": true, + "error": null, + "mrr": 0.5988452380952382, + "hit_at_1": 0.54, + "hit_at_5": 0.68, + "ndcg_at_10": 0.6323249003639557, + "p50_retrieval_latency_ms": 6.954291835427284, + "p95_retrieval_latency_ms": 7.816776633262634, + "avg_retrieval_latency_ms": 7.214257745072246, + "index_build_seconds": 0.004052711650729179, + "est_cost_per_query_usd": 0.0, + "generation_acc": null, + "p50_generation_latency_ms": null, + "avg_generation_latency_ms": null, + "generation_tokens_per_sec": null, + "est_generation_cost_per_query_usd": null, + "paraphrase_drop": 0.07990079365079361, + "shift_drop": 0.3071190476190476, + "n_queries": 100 + }, + { + "candidate_id": "magistral-small::hybrid::none", + "benchmark": "dataset-nucleotidings-qa", + "model": "magistral-small", + "retriever": "hybrid", + "reranker": "none", + "is_neon": true, + "error": null, + "mrr": 0.048261904761904756, + "hit_at_1": 0.01, + "hit_at_5": 0.1, + "ndcg_at_10": 0.06569076601728688, + "p50_retrieval_latency_ms": 7.779561914503574, + "p95_retrieval_latency_ms": 8.675494231283665, + "avg_retrieval_latency_ms": 7.911766804754734, + "index_build_seconds": 0.0325782485306263, + "est_cost_per_query_usd": 0.0, + "generation_acc": null, + "p50_generation_latency_ms": null, + "avg_generation_latency_ms": null, + "generation_tokens_per_sec": null, + "est_generation_cost_per_query_usd": null, + "paraphrase_drop": 0.015111111111111103, + "shift_drop": 0.048261904761904756, + "n_queries": 100 + }, + { + "candidate_id": "magistral-small::hybrid::none", + "benchmark": "dataset-logistics-qa", + "model": "magistral-small", + "retriever": "hybrid", + "reranker": "none", + "is_neon": true, + "error": null, + "mrr": 1.0, + "hit_at_1": 1.0, + "hit_at_5": 1.0, + "ndcg_at_10": 1.0, + "p50_retrieval_latency_ms": 6.913617253303528, + "p95_retrieval_latency_ms": 7.607226260006428, + "avg_retrieval_latency_ms": 7.0380013063549995, + "index_build_seconds": 0.009966623038053513, + "est_cost_per_query_usd": 0.0, + "generation_acc": null, + "p50_generation_latency_ms": null, + "avg_generation_latency_ms": null, + "generation_tokens_per_sec": null, + "est_generation_cost_per_query_usd": null, + "paraphrase_drop": 0.0, + "shift_drop": 0.0, + "n_queries": 100 + }, + { + "candidate_id": "magistral-small::hybrid::none", + "benchmark": "dataset-landuse-wa-qa", + "model": "magistral-small", + "retriever": "hybrid", + "reranker": "none", + "is_neon": true, + "error": null, + "mrr": 1.0, + "hit_at_1": 1.0, + "hit_at_5": 1.0, + "ndcg_at_10": 1.0, + "p50_retrieval_latency_ms": 7.033617235720158, + "p95_retrieval_latency_ms": 7.600724697113037, + "avg_retrieval_latency_ms": 7.124365046620369, + "index_build_seconds": 0.007180897518992424, + "est_cost_per_query_usd": 0.0, + "generation_acc": null, + "p50_generation_latency_ms": null, + "avg_generation_latency_ms": null, + "generation_tokens_per_sec": null, + "est_generation_cost_per_query_usd": null, + "paraphrase_drop": 0.0, + "shift_drop": 1.0, + "n_queries": 100 + }, + { + "candidate_id": "magistral-small::hybrid::none", + "benchmark": "dataset-shakespeare-qa", + "model": "magistral-small", + "retriever": "hybrid", + "reranker": "none", + "is_neon": true, + "error": null, + "mrr": 1.0, + "hit_at_1": 1.0, + "hit_at_5": 1.0, + "ndcg_at_10": 1.0, + "p50_retrieval_latency_ms": 7.53379613161087, + "p95_retrieval_latency_ms": 7.8609399497509, + "avg_retrieval_latency_ms": 7.591790938749909, + "index_build_seconds": 0.05281056836247444, + "est_cost_per_query_usd": 0.0, + "generation_acc": null, + "p50_generation_latency_ms": null, + "avg_generation_latency_ms": null, + "generation_tokens_per_sec": null, + "est_generation_cost_per_query_usd": null, + "paraphrase_drop": 0.0, + "shift_drop": 0.0, + "n_queries": 100 + }, + { + "candidate_id": "magistral-small::hybrid::none", + "benchmark": "dataset-futurewise-qa", + "model": "magistral-small", + "retriever": "hybrid", + "reranker": "none", + "is_neon": true, + "error": null, + "mrr": 0.09780952380952382, + "hit_at_1": 0.03, + "hit_at_5": 0.19, + "ndcg_at_10": 0.13218233672282742, + "p50_retrieval_latency_ms": 8.058656938374043, + "p95_retrieval_latency_ms": 8.640003390610218, + "avg_retrieval_latency_ms": 8.11074884608388, + "index_build_seconds": 0.08653946872800589, + "est_cost_per_query_usd": 0.0, + "generation_acc": null, + "p50_generation_latency_ms": null, + "avg_generation_latency_ms": null, + "generation_tokens_per_sec": null, + "est_generation_cost_per_query_usd": null, + "paraphrase_drop": 0.03988095238095239, + "shift_drop": 0.04168707482993198, + "n_queries": 100 + }, + { + "candidate_id": "magistral-small::hybrid::none", + "benchmark": "dataset-belebele-qa", + "model": "magistral-small", + "retriever": "hybrid", + "reranker": "none", + "is_neon": true, + "error": null, + "mrr": 0.9678571428571429, + "hit_at_1": 0.95, + "hit_at_5": 0.9928571428571429, + "ndcg_at_10": 0.974179086645947, + "p50_retrieval_latency_ms": 7.476835511624813, + "p95_retrieval_latency_ms": 8.002723567187786, + "avg_retrieval_latency_ms": 7.445732723655445, + "index_build_seconds": 0.0043265325948596, + "est_cost_per_query_usd": 0.0, + "generation_acc": null, + "p50_generation_latency_ms": null, + "avg_generation_latency_ms": null, + "generation_tokens_per_sec": null, + "est_generation_cost_per_query_usd": null, + "paraphrase_drop": 0.06440476190476185, + "shift_drop": 0.0035714285714285587, + "n_queries": 140 + }, + { + "candidate_id": "devstral-small-24b::hybrid::none", + "benchmark": "dataset-neonai-qa", + "model": "devstral-small-24b", + "retriever": "hybrid", + "reranker": "none", + "is_neon": true, + "error": null, + "mrr": 0.5988452380952382, + "hit_at_1": 0.54, + "hit_at_5": 0.68, + "ndcg_at_10": 0.6323249003639557, + "p50_retrieval_latency_ms": 6.986231543123722, + "p95_retrieval_latency_ms": 7.795398123562336, + "avg_retrieval_latency_ms": 7.228139033541083, + "index_build_seconds": 0.003448634408414364, + "est_cost_per_query_usd": 0.0, + "generation_acc": null, + "p50_generation_latency_ms": null, + "avg_generation_latency_ms": null, + "generation_tokens_per_sec": null, + "est_generation_cost_per_query_usd": null, + "paraphrase_drop": 0.07990079365079361, + "shift_drop": 0.3071190476190476, + "n_queries": 100 + }, + { + "candidate_id": "devstral-small-24b::hybrid::none", + "benchmark": "dataset-nucleotidings-qa", + "model": "devstral-small-24b", + "retriever": "hybrid", + "reranker": "none", + "is_neon": true, + "error": null, + "mrr": 0.048261904761904756, + "hit_at_1": 0.01, + "hit_at_5": 0.1, + "ndcg_at_10": 0.06569076601728688, + "p50_retrieval_latency_ms": 7.648967206478119, + "p95_retrieval_latency_ms": 8.57258029282093, + "avg_retrieval_latency_ms": 7.856305995956063, + "index_build_seconds": 0.03500657342374325, + "est_cost_per_query_usd": 0.0, + "generation_acc": null, + "p50_generation_latency_ms": null, + "avg_generation_latency_ms": null, + "generation_tokens_per_sec": null, + "est_generation_cost_per_query_usd": null, + "paraphrase_drop": 0.015111111111111103, + "shift_drop": 0.048261904761904756, + "n_queries": 100 + }, + { + "candidate_id": "devstral-small-24b::hybrid::none", + "benchmark": "dataset-logistics-qa", + "model": "devstral-small-24b", + "retriever": "hybrid", + "reranker": "none", + "is_neon": true, + "error": null, + "mrr": 1.0, + "hit_at_1": 1.0, + "hit_at_5": 1.0, + "ndcg_at_10": 1.0, + "p50_retrieval_latency_ms": 7.002032361924648, + "p95_retrieval_latency_ms": 7.7148256823420525, + "avg_retrieval_latency_ms": 7.0997389033436775, + "index_build_seconds": 0.01002301275730133, + "est_cost_per_query_usd": 0.0, + "generation_acc": null, + "p50_generation_latency_ms": null, + "avg_generation_latency_ms": null, + "generation_tokens_per_sec": null, + "est_generation_cost_per_query_usd": null, + "paraphrase_drop": 0.0, + "shift_drop": 0.0, + "n_queries": 100 + }, + { + "candidate_id": "devstral-small-24b::hybrid::none", + "benchmark": "dataset-landuse-wa-qa", + "model": "devstral-small-24b", + "retriever": "hybrid", + "reranker": "none", + "is_neon": true, + "error": null, + "mrr": 1.0, + "hit_at_1": 1.0, + "hit_at_5": 1.0, + "ndcg_at_10": 1.0, + "p50_retrieval_latency_ms": 6.944989785552025, + "p95_retrieval_latency_ms": 7.596067152917385, + "avg_retrieval_latency_ms": 7.056590598076582, + "index_build_seconds": 0.007299384102225304, + "est_cost_per_query_usd": 0.0, + "generation_acc": null, + "p50_generation_latency_ms": null, + "avg_generation_latency_ms": null, + "generation_tokens_per_sec": null, + "est_generation_cost_per_query_usd": null, + "paraphrase_drop": 0.0, + "shift_drop": 1.0, + "n_queries": 100 + }, + { + "candidate_id": "devstral-small-24b::hybrid::none", + "benchmark": "dataset-shakespeare-qa", + "model": "devstral-small-24b", + "retriever": "hybrid", + "reranker": "none", + "is_neon": true, + "error": null, + "mrr": 1.0, + "hit_at_1": 1.0, + "hit_at_5": 1.0, + "ndcg_at_10": 1.0, + "p50_retrieval_latency_ms": 7.386316545307636, + "p95_retrieval_latency_ms": 7.697527296841145, + "avg_retrieval_latency_ms": 7.466522827744484, + "index_build_seconds": 0.054405439645051956, + "est_cost_per_query_usd": 0.0, + "generation_acc": null, + "p50_generation_latency_ms": null, + "avg_generation_latency_ms": null, + "generation_tokens_per_sec": null, + "est_generation_cost_per_query_usd": null, + "paraphrase_drop": 0.0, + "shift_drop": 0.0, + "n_queries": 100 + }, + { + "candidate_id": "devstral-small-24b::hybrid::none", + "benchmark": "dataset-futurewise-qa", + "model": "devstral-small-24b", + "retriever": "hybrid", + "reranker": "none", + "is_neon": true, + "error": null, + "mrr": 0.09780952380952382, + "hit_at_1": 0.03, + "hit_at_5": 0.19, + "ndcg_at_10": 0.13218233672282742, + "p50_retrieval_latency_ms": 7.762560620903969, + "p95_retrieval_latency_ms": 8.184201084077358, + "avg_retrieval_latency_ms": 7.790521765127778, + "index_build_seconds": 0.08099719695746899, + "est_cost_per_query_usd": 0.0, + "generation_acc": null, + "p50_generation_latency_ms": null, + "avg_generation_latency_ms": null, + "generation_tokens_per_sec": null, + "est_generation_cost_per_query_usd": null, + "paraphrase_drop": 0.03988095238095239, + "shift_drop": 0.04168707482993198, + "n_queries": 100 + }, + { + "candidate_id": "devstral-small-24b::hybrid::none", + "benchmark": "dataset-belebele-qa", + "model": "devstral-small-24b", + "retriever": "hybrid", + "reranker": "none", + "is_neon": true, + "error": null, + "mrr": 0.9678571428571429, + "hit_at_1": 0.95, + "hit_at_5": 0.9928571428571429, + "ndcg_at_10": 0.974179086645947, + "p50_retrieval_latency_ms": 7.34444335103035, + "p95_retrieval_latency_ms": 7.850646041333675, + "avg_retrieval_latency_ms": 7.294278585218957, + "index_build_seconds": 0.0038021234795451164, + "est_cost_per_query_usd": 0.0, + "generation_acc": null, + "p50_generation_latency_ms": null, + "avg_generation_latency_ms": null, + "generation_tokens_per_sec": null, + "est_generation_cost_per_query_usd": null, + "paraphrase_drop": 0.06440476190476185, + "shift_drop": 0.0035714285714285587, + "n_queries": 140 + }, + { + "candidate_id": "mistral-small-3.2-24b::hybrid::none", + "benchmark": "dataset-neonai-qa", + "model": "mistral-small-3.2-24b", + "retriever": "hybrid", + "reranker": "none", + "is_neon": true, + "error": null, + "mrr": 0.5988452380952382, + "hit_at_1": 0.54, + "hit_at_5": 0.68, + "ndcg_at_10": 0.6323249003639557, + "p50_retrieval_latency_ms": 7.042726501822472, + "p95_retrieval_latency_ms": 7.8193424269557, + "avg_retrieval_latency_ms": 7.272537685930729, + "index_build_seconds": 0.004021286033093929, + "est_cost_per_query_usd": 0.0, + "generation_acc": null, + "p50_generation_latency_ms": null, + "avg_generation_latency_ms": null, + "generation_tokens_per_sec": null, + "est_generation_cost_per_query_usd": null, + "paraphrase_drop": 0.07990079365079361, + "shift_drop": 0.3071190476190476, + "n_queries": 100 + }, + { + "candidate_id": "mistral-small-3.2-24b::hybrid::none", + "benchmark": "dataset-nucleotidings-qa", + "model": "mistral-small-3.2-24b", + "retriever": "hybrid", + "reranker": "none", + "is_neon": true, + "error": null, + "mrr": 0.048261904761904756, + "hit_at_1": 0.01, + "hit_at_5": 0.1, + "ndcg_at_10": 0.06569076601728688, + "p50_retrieval_latency_ms": 7.691754028201103, + "p95_retrieval_latency_ms": 8.569280616939068, + "avg_retrieval_latency_ms": 7.879103012382984, + "index_build_seconds": 0.033108511939644814, + "est_cost_per_query_usd": 0.0, + "generation_acc": null, + "p50_generation_latency_ms": null, + "avg_generation_latency_ms": null, + "generation_tokens_per_sec": null, + "est_generation_cost_per_query_usd": null, + "paraphrase_drop": 0.015111111111111103, + "shift_drop": 0.048261904761904756, + "n_queries": 100 + }, + { + "candidate_id": "mistral-small-3.2-24b::hybrid::none", + "benchmark": "dataset-logistics-qa", + "model": "mistral-small-3.2-24b", + "retriever": "hybrid", + "reranker": "none", + "is_neon": true, + "error": null, + "mrr": 1.0, + "hit_at_1": 1.0, + "hit_at_5": 1.0, + "ndcg_at_10": 1.0, + "p50_retrieval_latency_ms": 6.975045427680016, + "p95_retrieval_latency_ms": 7.641519419848919, + "avg_retrieval_latency_ms": 7.097518444061279, + "index_build_seconds": 0.010510171763598919, + "est_cost_per_query_usd": 0.0, + "generation_acc": null, + "p50_generation_latency_ms": null, + "avg_generation_latency_ms": null, + "generation_tokens_per_sec": null, + "est_generation_cost_per_query_usd": null, + "paraphrase_drop": 0.0, + "shift_drop": 0.0, + "n_queries": 100 + }, + { + "candidate_id": "mistral-small-3.2-24b::hybrid::none", + "benchmark": "dataset-landuse-wa-qa", + "model": "mistral-small-3.2-24b", + "retriever": "hybrid", + "reranker": "none", + "is_neon": true, + "error": null, + "mrr": 1.0, + "hit_at_1": 1.0, + "hit_at_5": 1.0, + "ndcg_at_10": 1.0, + "p50_retrieval_latency_ms": 7.0530083030462265, + "p95_retrieval_latency_ms": 7.611000910401344, + "avg_retrieval_latency_ms": 7.2303250059485435, + "index_build_seconds": 0.008502699434757233, + "est_cost_per_query_usd": 0.0, + "generation_acc": null, + "p50_generation_latency_ms": null, + "avg_generation_latency_ms": null, + "generation_tokens_per_sec": null, + "est_generation_cost_per_query_usd": null, + "paraphrase_drop": 0.0, + "shift_drop": 1.0, + "n_queries": 100 + }, + { + "candidate_id": "mistral-small-3.2-24b::hybrid::none", + "benchmark": "dataset-shakespeare-qa", + "model": "mistral-small-3.2-24b", + "retriever": "hybrid", + "reranker": "none", + "is_neon": true, + "error": null, + "mrr": 1.0, + "hit_at_1": 1.0, + "hit_at_5": 1.0, + "ndcg_at_10": 1.0, + "p50_retrieval_latency_ms": 7.434654980897903, + "p95_retrieval_latency_ms": 7.705735974013805, + "avg_retrieval_latency_ms": 7.511973651126027, + "index_build_seconds": 0.053165786899626255, + "est_cost_per_query_usd": 0.0, + "generation_acc": null, + "p50_generation_latency_ms": null, + "avg_generation_latency_ms": null, + "generation_tokens_per_sec": null, + "est_generation_cost_per_query_usd": null, + "paraphrase_drop": 0.0, + "shift_drop": 0.0, + "n_queries": 100 + }, + { + "candidate_id": "mistral-small-3.2-24b::hybrid::none", + "benchmark": "dataset-futurewise-qa", + "model": "mistral-small-3.2-24b", + "retriever": "hybrid", + "reranker": "none", + "is_neon": true, + "error": null, + "mrr": 0.09780952380952382, + "hit_at_1": 0.03, + "hit_at_5": 0.19, + "ndcg_at_10": 0.13218233672282742, + "p50_retrieval_latency_ms": 8.097118698060513, + "p95_retrieval_latency_ms": 8.389534428715706, + "avg_retrieval_latency_ms": 8.127077231183648, + "index_build_seconds": 0.08655050490051508, + "est_cost_per_query_usd": 0.0, + "generation_acc": null, + "p50_generation_latency_ms": null, + "avg_generation_latency_ms": null, + "generation_tokens_per_sec": null, + "est_generation_cost_per_query_usd": null, + "paraphrase_drop": 0.03988095238095239, + "shift_drop": 0.04168707482993198, + "n_queries": 100 + }, + { + "candidate_id": "mistral-small-3.2-24b::hybrid::none", + "benchmark": "dataset-belebele-qa", + "model": "mistral-small-3.2-24b", + "retriever": "hybrid", + "reranker": "none", + "is_neon": true, + "error": null, + "mrr": 0.9678571428571429, + "hit_at_1": 0.95, + "hit_at_5": 0.9928571428571429, + "ndcg_at_10": 0.974179086645947, + "p50_retrieval_latency_ms": 7.46503658592701, + "p95_retrieval_latency_ms": 7.775686681270599, + "avg_retrieval_latency_ms": 7.389138266444206, + "index_build_seconds": 0.004063928499817848, + "est_cost_per_query_usd": 0.0, + "generation_acc": null, + "p50_generation_latency_ms": null, + "avg_generation_latency_ms": null, + "generation_tokens_per_sec": null, + "est_generation_cost_per_query_usd": null, + "paraphrase_drop": 0.06440476190476185, + "shift_drop": 0.0035714285714285587, + "n_queries": 140 + }, + { + "candidate_id": "granite-4-30b::hybrid::none", + "benchmark": "dataset-neonai-qa", + "model": "granite-4-30b", + "retriever": "hybrid", + "reranker": "none", + "is_neon": true, + "error": null, + "mrr": 0.5988452380952382, + "hit_at_1": 0.54, + "hit_at_5": 0.68, + "ndcg_at_10": 0.6323249003639557, + "p50_retrieval_latency_ms": 7.0448704063892365, + "p95_retrieval_latency_ms": 7.770040072500706, + "avg_retrieval_latency_ms": 7.286299178376794, + "index_build_seconds": 0.0033295685425400734, + "est_cost_per_query_usd": 0.0, + "generation_acc": null, + "p50_generation_latency_ms": null, + "avg_generation_latency_ms": null, + "generation_tokens_per_sec": null, + "est_generation_cost_per_query_usd": null, + "paraphrase_drop": 0.07990079365079361, + "shift_drop": 0.3071190476190476, + "n_queries": 100 + }, + { + "candidate_id": "granite-4-30b::hybrid::none", + "benchmark": "dataset-nucleotidings-qa", + "model": "granite-4-30b", + "retriever": "hybrid", + "reranker": "none", + "is_neon": true, + "error": null, + "mrr": 0.048261904761904756, + "hit_at_1": 0.01, + "hit_at_5": 0.1, + "ndcg_at_10": 0.06569076601728688, + "p50_retrieval_latency_ms": 7.591321133077145, + "p95_retrieval_latency_ms": 8.210630156099796, + "avg_retrieval_latency_ms": 7.772924480959773, + "index_build_seconds": 0.03287146147340536, + "est_cost_per_query_usd": 0.0, + "generation_acc": null, + "p50_generation_latency_ms": null, + "avg_generation_latency_ms": null, + "generation_tokens_per_sec": null, + "est_generation_cost_per_query_usd": null, + "paraphrase_drop": 0.015111111111111103, + "shift_drop": 0.048261904761904756, + "n_queries": 100 + }, + { + "candidate_id": "granite-4-30b::hybrid::none", + "benchmark": "dataset-logistics-qa", + "model": "granite-4-30b", + "retriever": "hybrid", + "reranker": "none", + "is_neon": true, + "error": null, + "mrr": 1.0, + "hit_at_1": 1.0, + "hit_at_5": 1.0, + "ndcg_at_10": 1.0, + "p50_retrieval_latency_ms": 6.952958181500435, + "p95_retrieval_latency_ms": 7.622065953910351, + "avg_retrieval_latency_ms": 7.116861520335078, + "index_build_seconds": 0.009383691474795341, + "est_cost_per_query_usd": 0.0, + "generation_acc": null, + "p50_generation_latency_ms": null, + "avg_generation_latency_ms": null, + "generation_tokens_per_sec": null, + "est_generation_cost_per_query_usd": null, + "paraphrase_drop": 0.0, + "shift_drop": 0.0, + "n_queries": 100 + }, + { + "candidate_id": "granite-4-30b::hybrid::none", + "benchmark": "dataset-landuse-wa-qa", + "model": "granite-4-30b", + "retriever": "hybrid", + "reranker": "none", + "is_neon": true, + "error": null, + "mrr": 1.0, + "hit_at_1": 1.0, + "hit_at_5": 1.0, + "ndcg_at_10": 1.0, + "p50_retrieval_latency_ms": 7.011697627604008, + "p95_retrieval_latency_ms": 7.574110291898251, + "avg_retrieval_latency_ms": 7.0873454213142395, + "index_build_seconds": 0.008607922121882439, + "est_cost_per_query_usd": 0.0, + "generation_acc": null, + "p50_generation_latency_ms": null, + "avg_generation_latency_ms": null, + "generation_tokens_per_sec": null, + "est_generation_cost_per_query_usd": null, + "paraphrase_drop": 0.0, + "shift_drop": 1.0, + "n_queries": 100 + }, + { + "candidate_id": "granite-4-30b::hybrid::none", + "benchmark": "dataset-shakespeare-qa", + "model": "granite-4-30b", + "retriever": "hybrid", + "reranker": "none", + "is_neon": true, + "error": null, + "mrr": 1.0, + "hit_at_1": 1.0, + "hit_at_5": 1.0, + "ndcg_at_10": 1.0, + "p50_retrieval_latency_ms": 7.527725771069527, + "p95_retrieval_latency_ms": 7.810373790562153, + "avg_retrieval_latency_ms": 7.593083288520575, + "index_build_seconds": 0.052798391319811344, + "est_cost_per_query_usd": 0.0, + "generation_acc": null, + "p50_generation_latency_ms": null, + "avg_generation_latency_ms": null, + "generation_tokens_per_sec": null, + "est_generation_cost_per_query_usd": null, + "paraphrase_drop": 0.0, + "shift_drop": 0.0, + "n_queries": 100 + }, + { + "candidate_id": "granite-4-30b::hybrid::none", + "benchmark": "dataset-futurewise-qa", + "model": "granite-4-30b", + "retriever": "hybrid", + "reranker": "none", + "is_neon": true, + "error": null, + "mrr": 0.09780952380952382, + "hit_at_1": 0.03, + "hit_at_5": 0.19, + "ndcg_at_10": 0.13218233672282742, + "p50_retrieval_latency_ms": 7.9466793686151505, + "p95_retrieval_latency_ms": 8.466493338346481, + "avg_retrieval_latency_ms": 7.993034860119224, + "index_build_seconds": 0.08808730728924274, + "est_cost_per_query_usd": 0.0, + "generation_acc": null, + "p50_generation_latency_ms": null, + "avg_generation_latency_ms": null, + "generation_tokens_per_sec": null, + "est_generation_cost_per_query_usd": null, + "paraphrase_drop": 0.03988095238095239, + "shift_drop": 0.04168707482993198, + "n_queries": 100 + }, + { + "candidate_id": "granite-4-30b::hybrid::none", + "benchmark": "dataset-belebele-qa", + "model": "granite-4-30b", + "retriever": "hybrid", + "reranker": "none", + "is_neon": true, + "error": null, + "mrr": 0.9678571428571429, + "hit_at_1": 0.95, + "hit_at_5": 0.9928571428571429, + "ndcg_at_10": 0.974179086645947, + "p50_retrieval_latency_ms": 7.436822168529034, + "p95_retrieval_latency_ms": 7.707064040005207, + "avg_retrieval_latency_ms": 7.361321929576142, + "index_build_seconds": 0.004397517070174217, + "est_cost_per_query_usd": 0.0, + "generation_acc": null, + "p50_generation_latency_ms": null, + "avg_generation_latency_ms": null, + "generation_tokens_per_sec": null, + "est_generation_cost_per_query_usd": null, + "paraphrase_drop": 0.06440476190476185, + "shift_drop": 0.0035714285714285587, + "n_queries": 140 + } + ], + "profiles": [ + { + "name": "enterprise_news", + "metric": "mrr", + "gates": [ + "p50 ≤ 200 ms", + "$/q ≤ 0.001", + "MRR ≥ 0.85" + ], + "corpus_profile": "news_articles", + "query_mix": { + "extractive": 0.7, + "paraphrased": 0.25, + "multi_hop": 0.05 + }, + "deployment_constraint": "single_gpu_10gb", + "quality_weight": 0.7 + }, + { + "name": "language_following", + "metric": "generation_acc", + "gates": [ + "p50 ≤ 60000 ms", + "gen ≤ 12000 ms", + "$/q ≤ 0.02", + "gen ≥ 0.5" + ], + "corpus_profile": "mixed_html", + "query_mix": { + "extractive": 0.5, + "paraphrased": 0.3, + "multi_hop": 0.2 + }, + "deployment_constraint": "multi_gpu", + "quality_weight": 0.9 + }, + { + "name": "model_efficiency", + "metric": "generation_acc", + "gates": [ + "p50 ≤ 500 ms", + "gen ≤ 8000 ms", + "$/q ≤ 0.01", + "gen ≥ 0.85" + ], + "corpus_profile": "technical_papers", + "query_mix": { + "extractive": 0.4, + "paraphrased": 0.4, + "multi_hop": 0.2 + }, + "deployment_constraint": "multi_gpu", + "quality_weight": 0.65 + }, + { + "name": "multilingual_qa", + "metric": "generation_acc", + "gates": [ + "p50 ≤ 1500 ms", + "gen ≤ 12000 ms", + "$/q ≤ 0.02", + "gen ≥ 0.25" + ], + "corpus_profile": "mixed_html", + "query_mix": { + "extractive": 0.5, + "paraphrased": 0.3, + "multi_hop": 0.2 + }, + "deployment_constraint": "multi_gpu", + "quality_weight": 0.7 + }, + { + "name": "science_qa", + "metric": "mrr", + "gates": [ + "p50 ≤ 1500 ms", + "$/q ≤ 0.005", + "MRR ≥ 0.8" + ], + "corpus_profile": "technical_papers", + "query_mix": { + "extractive": 0.3, + "paraphrased": 0.4, + "multi_hop": 0.3 + }, + "deployment_constraint": "single_gpu_10gb", + "quality_weight": 0.9 + } + ], + "picks": { + "dataset-neonai-qa": { + "enterprise_news": { + "pick": null, + "verdict": "all disqualified" + }, + "language_following": { + "pick": "devstral-123b::hybrid::none", + "model": "devstral-123b", + "retriever": "hybrid", + "reranker": "none", + "mrr": 0.5988452380952382, + "metric": "generation_acc", + "metric_val": 0.92, + "p50": 7.858959957957268, + "verdict": "winner clear" + }, + "model_efficiency": { + "pick": "devstral-123b::hybrid::none", + "model": "devstral-123b", + "retriever": "hybrid", + "reranker": "none", + "mrr": 0.5988452380952382, + "metric": "generation_acc", + "metric_val": 0.92, + "p50": 7.858959957957268, + "verdict": "winner clear" + }, + "multilingual_qa": { + "pick": "devstral-123b::hybrid::none", + "model": "devstral-123b", + "retriever": "hybrid", + "reranker": "none", + "mrr": 0.5988452380952382, + "metric": "generation_acc", + "metric_val": 0.92, + "p50": 7.858959957957268, + "verdict": "winner clear" + }, + "science_qa": { + "pick": null, + "verdict": "all disqualified" + } + }, + "dataset-nucleotidings-qa": { + "enterprise_news": { + "pick": null, + "verdict": "all disqualified" + }, + "language_following": { + "pick": "devstral-123b::hybrid::none", + "model": "devstral-123b", + "retriever": "hybrid", + "reranker": "none", + "mrr": 0.048261904761904756, + "metric": "generation_acc", + "metric_val": 0.66, + "p50": 8.428102359175682, + "verdict": "winner clear" + }, + "model_efficiency": { + "pick": "retrieval-only::bm25-only::none", + "model": "retrieval-only", + "retriever": "bm25-only", + "reranker": "none", + "mrr": 0.03176190476190476, + "metric": "generation_acc", + "metric_val": null, + "p50": 0.28319843113422394, + "verdict": "inconclusive" + }, + "multilingual_qa": { + "pick": "devstral-123b::hybrid::none", + "model": "devstral-123b", + "retriever": "hybrid", + "reranker": "none", + "mrr": 0.048261904761904756, + "metric": "generation_acc", + "metric_val": 0.66, + "p50": 8.428102359175682, + "verdict": "winner clear" + }, + "science_qa": { + "pick": null, + "verdict": "all disqualified" + } + }, + "dataset-logistics-qa": { + "enterprise_news": { + "pick": "magistral-small::hybrid::none", + "model": "magistral-small", + "retriever": "hybrid", + "reranker": "none", + "mrr": 1.0, + "metric": "mrr", + "metric_val": 1.0, + "p50": 6.913617253303528, + "verdict": "inconclusive" + }, + "language_following": { + "pick": "devstral-123b::hybrid::none", + "model": "devstral-123b", + "retriever": "hybrid", + "reranker": "none", + "mrr": 1.0, + "metric": "generation_acc", + "metric_val": 0.71, + "p50": 7.7611226588487625, + "verdict": "inconclusive" + }, + "model_efficiency": { + "pick": "magistral-small::hybrid::none", + "model": "magistral-small", + "retriever": "hybrid", + "reranker": "none", + "mrr": 1.0, + "metric": "generation_acc", + "metric_val": null, + "p50": 6.913617253303528, + "verdict": "inconclusive" + }, + "multilingual_qa": { + "pick": "devstral-123b::hybrid::none", + "model": "devstral-123b", + "retriever": "hybrid", + "reranker": "none", + "mrr": 1.0, + "metric": "generation_acc", + "metric_val": 0.71, + "p50": 7.7611226588487625, + "verdict": "inconclusive" + }, + "science_qa": { + "pick": "magistral-small::hybrid::none", + "model": "magistral-small", + "retriever": "hybrid", + "reranker": "none", + "mrr": 1.0, + "metric": "mrr", + "metric_val": 1.0, + "p50": 6.913617253303528, + "verdict": "inconclusive" + } + }, + "dataset-landuse-wa-qa": { + "enterprise_news": { + "pick": "devstral-small-24b::hybrid::none", + "model": "devstral-small-24b", + "retriever": "hybrid", + "reranker": "none", + "mrr": 1.0, + "metric": "mrr", + "metric_val": 1.0, + "p50": 6.944989785552025, + "verdict": "inconclusive" + }, + "language_following": { + "pick": "devstral-123b::hybrid::none", + "model": "devstral-123b", + "retriever": "hybrid", + "reranker": "none", + "mrr": 1.0, + "metric": "generation_acc", + "metric_val": 0.89, + "p50": 7.857654243707657, + "verdict": "inconclusive" + }, + "model_efficiency": { + "pick": "devstral-123b::hybrid::none", + "model": "devstral-123b", + "retriever": "hybrid", + "reranker": "none", + "mrr": 1.0, + "metric": "generation_acc", + "metric_val": 0.89, + "p50": 7.857654243707657, + "verdict": "inconclusive" + }, + "multilingual_qa": { + "pick": "devstral-123b::hybrid::none", + "model": "devstral-123b", + "retriever": "hybrid", + "reranker": "none", + "mrr": 1.0, + "metric": "generation_acc", + "metric_val": 0.89, + "p50": 7.857654243707657, + "verdict": "inconclusive" + }, + "science_qa": { + "pick": "devstral-small-24b::hybrid::none", + "model": "devstral-small-24b", + "retriever": "hybrid", + "reranker": "none", + "mrr": 1.0, + "metric": "mrr", + "metric_val": 1.0, + "p50": 6.944989785552025, + "verdict": "inconclusive" + } + }, + "dataset-shakespeare-qa": { + "enterprise_news": { + "pick": "devstral-small-24b::hybrid::none", + "model": "devstral-small-24b", + "retriever": "hybrid", + "reranker": "none", + "mrr": 1.0, + "metric": "mrr", + "metric_val": 1.0, + "p50": 7.386316545307636, + "verdict": "inconclusive" + }, + "language_following": { + "pick": "devstral-123b::hybrid::none", + "model": "devstral-123b", + "retriever": "hybrid", + "reranker": "none", + "mrr": 1.0, + "metric": "generation_acc", + "metric_val": 0.71, + "p50": 8.404421620070934, + "verdict": "inconclusive" + }, + "model_efficiency": { + "pick": "devstral-small-24b::hybrid::none", + "model": "devstral-small-24b", + "retriever": "hybrid", + "reranker": "none", + "mrr": 1.0, + "metric": "generation_acc", + "metric_val": null, + "p50": 7.386316545307636, + "verdict": "inconclusive" + }, + "multilingual_qa": { + "pick": "devstral-123b::hybrid::none", + "model": "devstral-123b", + "retriever": "hybrid", + "reranker": "none", + "mrr": 1.0, + "metric": "generation_acc", + "metric_val": 0.71, + "p50": 8.404421620070934, + "verdict": "inconclusive" + }, + "science_qa": { + "pick": "devstral-small-24b::hybrid::none", + "model": "devstral-small-24b", + "retriever": "hybrid", + "reranker": "none", + "mrr": 1.0, + "metric": "mrr", + "metric_val": 1.0, + "p50": 7.386316545307636, + "verdict": "inconclusive" + } + }, + "dataset-futurewise-qa": { + "enterprise_news": { + "pick": null, + "verdict": "all disqualified" + }, + "language_following": { + "pick": "devstral-123b::hybrid::none", + "model": "devstral-123b", + "retriever": "hybrid", + "reranker": "none", + "mrr": 0.09780952380952382, + "metric": "generation_acc", + "metric_val": 0.77, + "p50": 8.942721411585808, + "verdict": "winner clear" + }, + "model_efficiency": { + "pick": "devstral-small-24b::hybrid::none", + "model": "devstral-small-24b", + "retriever": "hybrid", + "reranker": "none", + "mrr": 0.09780952380952382, + "metric": "generation_acc", + "metric_val": null, + "p50": 7.762560620903969, + "verdict": "inconclusive" + }, + "multilingual_qa": { + "pick": "devstral-123b::hybrid::none", + "model": "devstral-123b", + "retriever": "hybrid", + "reranker": "none", + "mrr": 0.09780952380952382, + "metric": "generation_acc", + "metric_val": 0.77, + "p50": 8.942721411585808, + "verdict": "winner clear" + }, + "science_qa": { + "pick": null, + "verdict": "all disqualified" + } + }, + "dataset-belebele-qa": { + "enterprise_news": { + "pick": "devstral-small-24b::hybrid::none", + "model": "devstral-small-24b", + "retriever": "hybrid", + "reranker": "none", + "mrr": 0.9678571428571429, + "metric": "mrr", + "metric_val": 0.95, + "p50": 7.34444335103035, + "verdict": "inconclusive" + }, + "language_following": { + "pick": "devstral-123b::hybrid::none", + "model": "devstral-123b", + "retriever": "hybrid", + "reranker": "none", + "mrr": 0.9678571428571429, + "metric": "generation_acc", + "metric_val": 0.9, + "p50": 8.227682672441006, + "verdict": "inconclusive" + }, + "model_efficiency": { + "pick": "devstral-123b::hybrid::none", + "model": "devstral-123b", + "retriever": "hybrid", + "reranker": "none", + "mrr": 0.9678571428571429, + "metric": "generation_acc", + "metric_val": 0.9, + "p50": 8.227682672441006, + "verdict": "inconclusive" + }, + "multilingual_qa": { + "pick": "devstral-123b::hybrid::none", + "model": "devstral-123b", + "retriever": "hybrid", + "reranker": "none", + "mrr": 0.9678571428571429, + "metric": "generation_acc", + "metric_val": 0.9, + "p50": 8.227682672441006, + "verdict": "inconclusive" + }, + "science_qa": { + "pick": "devstral-small-24b::hybrid::none", + "model": "devstral-small-24b", + "retriever": "hybrid", + "reranker": "none", + "mrr": 0.9678571428571429, + "metric": "mrr", + "metric_val": 0.95, + "p50": 7.34444335103035, + "verdict": "inconclusive" + } + } + }, + "bench_tables": { + "dataset-neonai-qa": { + "order": [ + "retrieval-only::hybrid::none", + "retrieval-only::hybrid-web::none", + "devstral-123b::hybrid::none", + "ministral-14b-reasoning::hybrid::none", + "magistral-small::hybrid::none", + "devstral-small-24b::hybrid::none", + "mistral-small-3.2-24b::hybrid::none", + "granite-4-30b::hybrid::none", + "retrieval-only::hybrid-triple::none", + "retrieval-only::hybrid-bge-large::none", + "retrieval-only::hybrid-bge-large::minilm", + "retrieval-only::bm25-only::minilm", + "retrieval-only::hybrid::minilm", + "retrieval-only::hybrid-web::minilm", + "retrieval-only::hybrid-triple::minilm", + "retrieval-only::splade-only::minilm", + "retrieval-only::embedding-bge-large::minilm", + "retrieval-only::embedding-bge-large::none", + "retrieval-only::embedding-bge-m3::minilm", + "retrieval-only::embedding-bge-m3::none", + "retrieval-only::hybrid::mxbai", + "retrieval-only::hybrid-web::mxbai", + "retrieval-only::hybrid-bge-large::mxbai", + "retrieval-only::hybrid-triple::mxbai", + "retrieval-only::bm25-only::none", + "retrieval-only::embedding-bge-large::mxbai", + "retrieval-only::splade-only::none", + "retrieval-only::embedding-bge-m3::mxbai", + "retrieval-only::bm25-only::mxbai", + "retrieval-only::splade-only::mxbai" + ], + "pareto": [ + "devstral-123b::hybrid::none", + "devstral-small-24b::hybrid::none", + "granite-4-30b::hybrid::none", + "magistral-small::hybrid::none", + "retrieval-only::bm25-only::none", + "retrieval-only::hybrid-bge-large::none", + "retrieval-only::hybrid::none", + "retrieval-only::splade-only::minilm", + "retrieval-only::splade-only::mxbai", + "retrieval-only::splade-only::none" + ], + "dq": { + "retrieval-only::hybrid::none": "accuracy", + "retrieval-only::hybrid-web::none": "latency", + "devstral-123b::hybrid::none": "cost", + "ministral-14b-reasoning::hybrid::none": "accuracy", + "magistral-small::hybrid::none": "accuracy", + "devstral-small-24b::hybrid::none": "accuracy", + "mistral-small-3.2-24b::hybrid::none": "accuracy", + "granite-4-30b::hybrid::none": "accuracy", + "retrieval-only::hybrid-triple::none": "accuracy", + "retrieval-only::hybrid-bge-large::none": "accuracy", + "retrieval-only::hybrid-bge-large::minilm": "accuracy", + "retrieval-only::bm25-only::minilm": "accuracy", + "retrieval-only::hybrid::minilm": "accuracy", + "retrieval-only::hybrid-web::minilm": "latency", + "retrieval-only::hybrid-triple::minilm": "accuracy", + "retrieval-only::splade-only::minilm": "accuracy", + "retrieval-only::embedding-bge-large::minilm": "accuracy", + "retrieval-only::embedding-bge-large::none": "accuracy", + "retrieval-only::embedding-bge-m3::minilm": "accuracy", + "retrieval-only::embedding-bge-m3::none": "accuracy", + "retrieval-only::hybrid::mxbai": "latency", + "retrieval-only::hybrid-web::mxbai": "latency", + "retrieval-only::hybrid-bge-large::mxbai": "latency", + "retrieval-only::hybrid-triple::mxbai": "latency", + "retrieval-only::bm25-only::none": "accuracy", + "retrieval-only::embedding-bge-large::mxbai": "latency", + "retrieval-only::splade-only::none": "accuracy", + "retrieval-only::embedding-bge-m3::mxbai": "latency", + "retrieval-only::bm25-only::mxbai": "latency", + "retrieval-only::splade-only::mxbai": "latency" + } + }, + "dataset-nucleotidings-qa": { + "order": [ + "retrieval-only::hybrid-bge-large::minilm", + "retrieval-only::hybrid-triple::minilm", + "retrieval-only::hybrid::minilm", + "retrieval-only::hybrid-web::minilm", + "retrieval-only::hybrid-bge-large::mxbai", + "retrieval-only::hybrid-triple::none", + "retrieval-only::splade-only::minilm", + "retrieval-only::embedding-bge-m3::minilm", + "retrieval-only::hybrid::mxbai", + "retrieval-only::hybrid-web::mxbai", + "retrieval-only::hybrid-triple::mxbai", + "retrieval-only::bm25-only::minilm", + "retrieval-only::embedding-bge-m3::mxbai", + "retrieval-only::splade-only::mxbai", + "retrieval-only::hybrid::none", + "retrieval-only::hybrid-web::none", + "devstral-123b::hybrid::none", + "ministral-14b-reasoning::hybrid::none", + "magistral-small::hybrid::none", + "devstral-small-24b::hybrid::none", + "mistral-small-3.2-24b::hybrid::none", + "granite-4-30b::hybrid::none", + "retrieval-only::embedding-bge-large::mxbai", + "retrieval-only::embedding-bge-large::minilm", + "retrieval-only::splade-only::none", + "retrieval-only::bm25-only::mxbai", + "retrieval-only::embedding-bge-m3::none", + "retrieval-only::bm25-only::none", + "retrieval-only::embedding-bge-large::none", + "retrieval-only::hybrid-bge-large::none" + ], + "pareto": [ + "devstral-123b::hybrid::none", + "retrieval-only::bm25-only::minilm", + "retrieval-only::bm25-only::none", + "retrieval-only::embedding-bge-m3::minilm", + "retrieval-only::hybrid-bge-large::minilm", + "retrieval-only::hybrid-triple::none", + "retrieval-only::hybrid::minilm", + "retrieval-only::hybrid::none", + "retrieval-only::splade-only::minilm", + "retrieval-only::splade-only::none" + ], + "dq": { + "retrieval-only::hybrid-bge-large::minilm": "accuracy", + "retrieval-only::hybrid-triple::minilm": "accuracy", + "retrieval-only::hybrid::minilm": "accuracy", + "retrieval-only::hybrid-web::minilm": "latency", + "retrieval-only::hybrid-bge-large::mxbai": "latency", + "retrieval-only::hybrid-triple::none": "accuracy", + "retrieval-only::splade-only::minilm": "accuracy", + "retrieval-only::embedding-bge-m3::minilm": "accuracy", + "retrieval-only::hybrid::mxbai": "latency", + "retrieval-only::hybrid-web::mxbai": "latency", + "retrieval-only::hybrid-triple::mxbai": "latency", + "retrieval-only::bm25-only::minilm": "accuracy", + "retrieval-only::embedding-bge-m3::mxbai": "latency", + "retrieval-only::splade-only::mxbai": "latency", + "retrieval-only::hybrid::none": "accuracy", + "retrieval-only::hybrid-web::none": "latency", + "devstral-123b::hybrid::none": "cost", + "ministral-14b-reasoning::hybrid::none": "accuracy", + "magistral-small::hybrid::none": "accuracy", + "devstral-small-24b::hybrid::none": "accuracy", + "mistral-small-3.2-24b::hybrid::none": "accuracy", + "granite-4-30b::hybrid::none": "accuracy", + "retrieval-only::embedding-bge-large::mxbai": "latency", + "retrieval-only::embedding-bge-large::minilm": "accuracy", + "retrieval-only::splade-only::none": "accuracy", + "retrieval-only::bm25-only::mxbai": "latency", + "retrieval-only::embedding-bge-m3::none": "accuracy", + "retrieval-only::bm25-only::none": "accuracy", + "retrieval-only::embedding-bge-large::none": "accuracy", + "retrieval-only::hybrid-bge-large::none": "accuracy" + } + }, + "dataset-logistics-qa": { + "order": [ + "devstral-123b::hybrid::none", + "ministral-14b-reasoning::hybrid::none", + "magistral-small::hybrid::none", + "devstral-small-24b::hybrid::none", + "mistral-small-3.2-24b::hybrid::none", + "granite-4-30b::hybrid::none" + ], + "pareto": [ + "magistral-small::hybrid::none", + "ministral-14b-reasoning::hybrid::none" + ], + "dq": { + "devstral-123b::hybrid::none": "cost" + } + }, + "dataset-landuse-wa-qa": { + "order": [ + "devstral-123b::hybrid::none", + "ministral-14b-reasoning::hybrid::none", + "magistral-small::hybrid::none", + "devstral-small-24b::hybrid::none", + "mistral-small-3.2-24b::hybrid::none", + "granite-4-30b::hybrid::none" + ], + "pareto": [ + "devstral-small-24b::hybrid::none", + "magistral-small::hybrid::none" + ], + "dq": { + "devstral-123b::hybrid::none": "cost" + } + }, + "dataset-shakespeare-qa": { + "order": [ + "devstral-123b::hybrid::none", + "ministral-14b-reasoning::hybrid::none", + "magistral-small::hybrid::none", + "devstral-small-24b::hybrid::none", + "mistral-small-3.2-24b::hybrid::none", + "granite-4-30b::hybrid::none" + ], + "pareto": [ + "devstral-123b::hybrid::none", + "devstral-small-24b::hybrid::none", + "granite-4-30b::hybrid::none", + "magistral-small::hybrid::none", + "mistral-small-3.2-24b::hybrid::none" + ], + "dq": { + "devstral-123b::hybrid::none": "accuracy" + } + }, + "dataset-futurewise-qa": { + "order": [ + "devstral-123b::hybrid::none", + "ministral-14b-reasoning::hybrid::none", + "magistral-small::hybrid::none", + "devstral-small-24b::hybrid::none", + "mistral-small-3.2-24b::hybrid::none", + "granite-4-30b::hybrid::none" + ], + "pareto": [ + "devstral-123b::hybrid::none", + "devstral-small-24b::hybrid::none", + "ministral-14b-reasoning::hybrid::none" + ], + "dq": { + "devstral-123b::hybrid::none": "accuracy", + "ministral-14b-reasoning::hybrid::none": "accuracy", + "magistral-small::hybrid::none": "accuracy", + "devstral-small-24b::hybrid::none": "accuracy", + "mistral-small-3.2-24b::hybrid::none": "accuracy", + "granite-4-30b::hybrid::none": "accuracy" + } + }, + "dataset-belebele-qa": { + "order": [ + "devstral-123b::hybrid::none", + "ministral-14b-reasoning::hybrid::none", + "magistral-small::hybrid::none", + "devstral-small-24b::hybrid::none", + "mistral-small-3.2-24b::hybrid::none", + "granite-4-30b::hybrid::none" + ], + "pareto": [ + "devstral-small-24b::hybrid::none", + "ministral-14b-reasoning::hybrid::none" + ], + "dq": {} + } + }, + "metric_stats": { + "mrr": { + "retrieval-only::bm25-only::none": { + "wins": 0, + "losses": 0, + "count": 2, + "mean": 0.22009523809523818 + }, + "retrieval-only::bm25-only::minilm": { + "wins": 0, + "losses": 0, + "count": 2, + "mean": 0.2693194444444445 + }, + "retrieval-only::bm25-only::mxbai": { + "wins": 0, + "losses": 0, + "count": 2, + "mean": 0.209015873015873 + }, + "retrieval-only::embedding-bge-m3::none": { + "wins": 0, + "losses": 0, + "count": 2, + "mean": 0.2356309523809524 + }, + "retrieval-only::embedding-bge-m3::minilm": { + "wins": 0, + "losses": 0, + "count": 2, + "mean": 0.24796230158730168 + }, + "retrieval-only::embedding-bge-m3::mxbai": { + "wins": 0, + "losses": 0, + "count": 2, + "mean": 0.2262936507936508 + }, + "retrieval-only::embedding-bge-large::none": { + "wins": 0, + "losses": 0, + "count": 2, + "mean": 0.23683531746031752 + }, + "retrieval-only::embedding-bge-large::minilm": { + "wins": 0, + "losses": 0, + "count": 2, + "mean": 0.24646031746031757 + }, + "retrieval-only::embedding-bge-large::mxbai": { + "wins": 0, + "losses": 0, + "count": 2, + "mean": 0.2276051587301588 + }, + "retrieval-only::hybrid::none": { + "wins": 1, + "losses": 0, + "count": 2, + "mean": 0.32355357142857144 + }, + "retrieval-only::hybrid::minilm": { + "wins": 0, + "losses": 0, + "count": 2, + "mean": 0.2671051587301587 + }, + "retrieval-only::hybrid::mxbai": { + "wins": 0, + "losses": 0, + "count": 2, + "mean": 0.23812103174603175 + }, + "retrieval-only::hybrid-bge-large::none": { + "wins": 0, + "losses": 1, + "count": 2, + "mean": 0.3023313492063492 + }, + "retrieval-only::hybrid-bge-large::minilm": { + "wins": 1, + "losses": 0, + "count": 2, + "mean": 0.2783472222222223 + }, + "retrieval-only::hybrid-bge-large::mxbai": { + "wins": 0, + "losses": 0, + "count": 2, + "mean": 0.23511706349206357 + }, + "retrieval-only::splade-only::none": { + "wins": 0, + "losses": 0, + "count": 2, + "mean": 0.2220694444444445 + }, + "retrieval-only::splade-only::minilm": { + "wins": 0, + "losses": 0, + "count": 2, + "mean": 0.2552103174603175 + }, + "retrieval-only::splade-only::mxbai": { + "wins": 0, + "losses": 1, + "count": 2, + "mean": 0.2106309523809524 + }, + "retrieval-only::hybrid-triple::none": { + "wins": 0, + "losses": 0, + "count": 2, + "mean": 0.32097023809523817 + }, + "retrieval-only::hybrid-triple::minilm": { + "wins": 0, + "losses": 0, + "count": 2, + "mean": 0.2667063492063492 + }, + "retrieval-only::hybrid-triple::mxbai": { + "wins": 0, + "losses": 0, + "count": 2, + "mean": 0.2324484126984128 + }, + "retrieval-only::hybrid-web::none": { + "wins": 1, + "losses": 0, + "count": 2, + "mean": 0.32355357142857144 + }, + "retrieval-only::hybrid-web::minilm": { + "wins": 0, + "losses": 0, + "count": 2, + "mean": 0.2671051587301587 + }, + "retrieval-only::hybrid-web::mxbai": { + "wins": 0, + "losses": 0, + "count": 2, + "mean": 0.23812103174603175 + }, + "devstral-123b::hybrid::none": { + "wins": 6, + "losses": 0, + "count": 7, + "mean": 0.6732534013605443 + }, + "ministral-14b-reasoning::hybrid::none": { + "wins": 6, + "losses": 0, + "count": 7, + "mean": 0.6732534013605443 + }, + "magistral-small::hybrid::none": { + "wins": 6, + "losses": 0, + "count": 7, + "mean": 0.6732534013605443 + }, + "devstral-small-24b::hybrid::none": { + "wins": 6, + "losses": 0, + "count": 7, + "mean": 0.6732534013605443 + }, + "mistral-small-3.2-24b::hybrid::none": { + "wins": 6, + "losses": 0, + "count": 7, + "mean": 0.6732534013605443 + }, + "granite-4-30b::hybrid::none": { + "wins": 6, + "losses": 0, + "count": 7, + "mean": 0.6732534013605443 + } + }, + "h1": { + "retrieval-only::bm25-only::none": { + "wins": 0, + "losses": 1, + "count": 2, + "mean": 0.165 + }, + "retrieval-only::bm25-only::minilm": { + "wins": 0, + "losses": 0, + "count": 2, + "mean": 0.2 + }, + "retrieval-only::bm25-only::mxbai": { + "wins": 0, + "losses": 1, + "count": 2, + "mean": 0.14500000000000002 + }, + "retrieval-only::embedding-bge-m3::none": { + "wins": 0, + "losses": 0, + "count": 2, + "mean": 0.155 + }, + "retrieval-only::embedding-bge-m3::minilm": { + "wins": 0, + "losses": 0, + "count": 2, + "mean": 0.18 + }, + "retrieval-only::embedding-bge-m3::mxbai": { + "wins": 0, + "losses": 0, + "count": 2, + "mean": 0.165 + }, + "retrieval-only::embedding-bge-large::none": { + "wins": 0, + "losses": 1, + "count": 2, + "mean": 0.18 + }, + "retrieval-only::embedding-bge-large::minilm": { + "wins": 0, + "losses": 0, + "count": 2, + "mean": 0.17500000000000002 + }, + "retrieval-only::embedding-bge-large::mxbai": { + "wins": 0, + "losses": 0, + "count": 2, + "mean": 0.17 + }, + "retrieval-only::hybrid::none": { + "wins": 1, + "losses": 0, + "count": 2, + "mean": 0.275 + }, + "retrieval-only::hybrid::minilm": { + "wins": 0, + "losses": 0, + "count": 2, + "mean": 0.19 + }, + "retrieval-only::hybrid::mxbai": { + "wins": 1, + "losses": 0, + "count": 2, + "mean": 0.175 + }, + "retrieval-only::hybrid-bge-large::none": { + "wins": 0, + "losses": 1, + "count": 2, + "mean": 0.26 + }, + "retrieval-only::hybrid-bge-large::minilm": { + "wins": 0, + "losses": 0, + "count": 2, + "mean": 0.185 + }, + "retrieval-only::hybrid-bge-large::mxbai": { + "wins": 0, + "losses": 0, + "count": 2, + "mean": 0.16 + }, + "retrieval-only::splade-only::none": { + "wins": 0, + "losses": 1, + "count": 2, + "mean": 0.165 + }, + "retrieval-only::splade-only::minilm": { + "wins": 0, + "losses": 0, + "count": 2, + "mean": 0.19 + }, + "retrieval-only::splade-only::mxbai": { + "wins": 0, + "losses": 0, + "count": 2, + "mean": 0.155 + }, + "retrieval-only::hybrid-triple::none": { + "wins": 0, + "losses": 0, + "count": 2, + "mean": 0.27 + }, + "retrieval-only::hybrid-triple::minilm": { + "wins": 0, + "losses": 0, + "count": 2, + "mean": 0.19 + }, + "retrieval-only::hybrid-triple::mxbai": { + "wins": 1, + "losses": 0, + "count": 2, + "mean": 0.16999999999999998 + }, + "retrieval-only::hybrid-web::none": { + "wins": 1, + "losses": 0, + "count": 2, + "mean": 0.275 + }, + "retrieval-only::hybrid-web::minilm": { + "wins": 0, + "losses": 0, + "count": 2, + "mean": 0.19 + }, + "retrieval-only::hybrid-web::mxbai": { + "wins": 1, + "losses": 0, + "count": 2, + "mean": 0.175 + }, + "devstral-123b::hybrid::none": { + "wins": 6, + "losses": 0, + "count": 7, + "mean": 0.647142857142857 + }, + "ministral-14b-reasoning::hybrid::none": { + "wins": 6, + "losses": 0, + "count": 7, + "mean": 0.647142857142857 + }, + "magistral-small::hybrid::none": { + "wins": 6, + "losses": 0, + "count": 7, + "mean": 0.647142857142857 + }, + "devstral-small-24b::hybrid::none": { + "wins": 6, + "losses": 0, + "count": 7, + "mean": 0.647142857142857 + }, + "mistral-small-3.2-24b::hybrid::none": { + "wins": 6, + "losses": 0, + "count": 7, + "mean": 0.647142857142857 + }, + "granite-4-30b::hybrid::none": { + "wins": 6, + "losses": 0, + "count": 7, + "mean": 0.647142857142857 + } + }, + "h5": { + "retrieval-only::bm25-only::none": { + "wins": 0, + "losses": 1, + "count": 2, + "mean": 0.28500000000000003 + }, + "retrieval-only::bm25-only::minilm": { + "wins": 0, + "losses": 0, + "count": 2, + "mean": 0.355 + }, + "retrieval-only::bm25-only::mxbai": { + "wins": 0, + "losses": 0, + "count": 2, + "mean": 0.31499999999999995 + }, + "retrieval-only::embedding-bge-m3::none": { + "wins": 0, + "losses": 0, + "count": 2, + "mean": 0.345 + }, + "retrieval-only::embedding-bge-m3::minilm": { + "wins": 0, + "losses": 0, + "count": 2, + "mean": 0.35 + }, + "retrieval-only::embedding-bge-m3::mxbai": { + "wins": 0, + "losses": 0, + "count": 2, + "mean": 0.29500000000000004 + }, + "retrieval-only::embedding-bge-large::none": { + "wins": 0, + "losses": 0, + "count": 2, + "mean": 0.30500000000000005 + }, + "retrieval-only::embedding-bge-large::minilm": { + "wins": 0, + "losses": 0, + "count": 2, + "mean": 0.345 + }, + "retrieval-only::embedding-bge-large::mxbai": { + "wins": 0, + "losses": 0, + "count": 2, + "mean": 0.31499999999999995 + }, + "retrieval-only::hybrid::none": { + "wins": 1, + "losses": 0, + "count": 2, + "mean": 0.39 + }, + "retrieval-only::hybrid::minilm": { + "wins": 0, + "losses": 0, + "count": 2, + "mean": 0.355 + }, + "retrieval-only::hybrid::mxbai": { + "wins": 0, + "losses": 0, + "count": 2, + "mean": 0.315 + }, + "retrieval-only::hybrid-bge-large::none": { + "wins": 0, + "losses": 0, + "count": 2, + "mean": 0.36 + }, + "retrieval-only::hybrid-bge-large::minilm": { + "wins": 1, + "losses": 0, + "count": 2, + "mean": 0.4 + }, + "retrieval-only::hybrid-bge-large::mxbai": { + "wins": 0, + "losses": 0, + "count": 2, + "mean": 0.315 + }, + "retrieval-only::splade-only::none": { + "wins": 0, + "losses": 1, + "count": 2, + "mean": 0.29 + }, + "retrieval-only::splade-only::minilm": { + "wins": 0, + "losses": 0, + "count": 2, + "mean": 0.33999999999999997 + }, + "retrieval-only::splade-only::mxbai": { + "wins": 0, + "losses": 0, + "count": 2, + "mean": 0.29 + }, + "retrieval-only::hybrid-triple::none": { + "wins": 0, + "losses": 0, + "count": 2, + "mean": 0.38 + }, + "retrieval-only::hybrid-triple::minilm": { + "wins": 0, + "losses": 0, + "count": 2, + "mean": 0.37 + }, + "retrieval-only::hybrid-triple::mxbai": { + "wins": 0, + "losses": 0, + "count": 2, + "mean": 0.3 + }, + "retrieval-only::hybrid-web::none": { + "wins": 1, + "losses": 0, + "count": 2, + "mean": 0.39 + }, + "retrieval-only::hybrid-web::minilm": { + "wins": 0, + "losses": 0, + "count": 2, + "mean": 0.355 + }, + "retrieval-only::hybrid-web::mxbai": { + "wins": 0, + "losses": 0, + "count": 2, + "mean": 0.315 + }, + "devstral-123b::hybrid::none": { + "wins": 6, + "losses": 0, + "count": 7, + "mean": 0.7089795918367346 + }, + "ministral-14b-reasoning::hybrid::none": { + "wins": 6, + "losses": 0, + "count": 7, + "mean": 0.7089795918367346 + }, + "magistral-small::hybrid::none": { + "wins": 6, + "losses": 0, + "count": 7, + "mean": 0.7089795918367346 + }, + "devstral-small-24b::hybrid::none": { + "wins": 6, + "losses": 0, + "count": 7, + "mean": 0.7089795918367346 + }, + "mistral-small-3.2-24b::hybrid::none": { + "wins": 6, + "losses": 0, + "count": 7, + "mean": 0.7089795918367346 + }, + "granite-4-30b::hybrid::none": { + "wins": 6, + "losses": 0, + "count": 7, + "mean": 0.7089795918367346 + } + }, + "ndcg": { + "retrieval-only::bm25-only::none": { + "wins": 0, + "losses": 0, + "count": 2, + "mean": 0.2583013442462098 + }, + "retrieval-only::bm25-only::minilm": { + "wins": 0, + "losses": 0, + "count": 2, + "mean": 0.30734957235292343 + }, + "retrieval-only::bm25-only::mxbai": { + "wins": 0, + "losses": 0, + "count": 2, + "mean": 0.251605778206734 + }, + "retrieval-only::embedding-bge-m3::none": { + "wins": 0, + "losses": 0, + "count": 2, + "mean": 0.27830895027512786 + }, + "retrieval-only::embedding-bge-m3::minilm": { + "wins": 0, + "losses": 0, + "count": 2, + "mean": 0.28984281647385224 + }, + "retrieval-only::embedding-bge-m3::mxbai": { + "wins": 0, + "losses": 0, + "count": 2, + "mean": 0.2680014625660154 + }, + "retrieval-only::embedding-bge-large::none": { + "wins": 0, + "losses": 1, + "count": 2, + "mean": 0.2726846673676603 + }, + "retrieval-only::embedding-bge-large::minilm": { + "wins": 0, + "losses": 0, + "count": 2, + "mean": 0.2951497118077595 + }, + "retrieval-only::embedding-bge-large::mxbai": { + "wins": 0, + "losses": 0, + "count": 2, + "mean": 0.26683272244230594 + }, + "retrieval-only::hybrid::none": { + "wins": 1, + "losses": 0, + "count": 2, + "mean": 0.3490078331906213 + }, + "retrieval-only::hybrid::minilm": { + "wins": 0, + "losses": 0, + "count": 2, + "mean": 0.3124858976171995 + }, + "retrieval-only::hybrid::mxbai": { + "wins": 0, + "losses": 0, + "count": 2, + "mean": 0.2816296718223945 + }, + "retrieval-only::hybrid-bge-large::none": { + "wins": 0, + "losses": 0, + "count": 2, + "mean": 0.33097737940028366 + }, + "retrieval-only::hybrid-bge-large::minilm": { + "wins": 1, + "losses": 0, + "count": 2, + "mean": 0.33428433272211305 + }, + "retrieval-only::hybrid-bge-large::mxbai": { + "wins": 0, + "losses": 0, + "count": 2, + "mean": 0.2840729725949876 + }, + "retrieval-only::splade-only::none": { + "wins": 0, + "losses": 0, + "count": 2, + "mean": 0.261045061542026 + }, + "retrieval-only::splade-only::minilm": { + "wins": 0, + "losses": 0, + "count": 2, + "mean": 0.2997848863194258 + }, + "retrieval-only::splade-only::mxbai": { + "wins": 0, + "losses": 1, + "count": 2, + "mean": 0.2458770321015341 + }, + "retrieval-only::hybrid-triple::none": { + "wins": 0, + "losses": 0, + "count": 2, + "mean": 0.34810720388560956 + }, + "retrieval-only::hybrid-triple::minilm": { + "wins": 0, + "losses": 0, + "count": 2, + "mean": 0.31880769420680904 + }, + "retrieval-only::hybrid-triple::mxbai": { + "wins": 0, + "losses": 0, + "count": 2, + "mean": 0.2747457569398053 + }, + "retrieval-only::hybrid-web::none": { + "wins": 1, + "losses": 0, + "count": 2, + "mean": 0.3490078331906213 + }, + "retrieval-only::hybrid-web::minilm": { + "wins": 0, + "losses": 0, + "count": 2, + "mean": 0.3124858976171995 + }, + "retrieval-only::hybrid-web::mxbai": { + "wins": 0, + "losses": 0, + "count": 2, + "mean": 0.2816296718223945 + }, + "devstral-123b::hybrid::none": { + "wins": 6, + "losses": 0, + "count": 7, + "mean": 0.6863395842500024 + }, + "ministral-14b-reasoning::hybrid::none": { + "wins": 6, + "losses": 0, + "count": 7, + "mean": 0.6863395842500024 + }, + "magistral-small::hybrid::none": { + "wins": 6, + "losses": 0, + "count": 7, + "mean": 0.6863395842500024 + }, + "devstral-small-24b::hybrid::none": { + "wins": 6, + "losses": 0, + "count": 7, + "mean": 0.6863395842500024 + }, + "mistral-small-3.2-24b::hybrid::none": { + "wins": 6, + "losses": 0, + "count": 7, + "mean": 0.6863395842500024 + }, + "granite-4-30b::hybrid::none": { + "wins": 6, + "losses": 0, + "count": 7, + "mean": 0.6863395842500024 + } + }, + "p50": { + "retrieval-only::bm25-only::none": { + "wins": 2, + "losses": 0, + "count": 2, + "mean": 0.24090008810162544 + }, + "retrieval-only::bm25-only::minilm": { + "wins": 0, + "losses": 0, + "count": 2, + "mean": 22.695181891322136 + }, + "retrieval-only::bm25-only::mxbai": { + "wins": 0, + "losses": 0, + "count": 2, + "mean": 190.02285366877913 + }, + "retrieval-only::embedding-bge-m3::none": { + "wins": 0, + "losses": 0, + "count": 2, + "mean": 5.813977215439081 + }, + "retrieval-only::embedding-bge-m3::minilm": { + "wins": 0, + "losses": 0, + "count": 2, + "mean": 29.342605732381344 + }, + "retrieval-only::embedding-bge-m3::mxbai": { + "wins": 0, + "losses": 0, + "count": 2, + "mean": 203.89279909431934 + }, + "retrieval-only::embedding-bge-large::none": { + "wins": 0, + "losses": 0, + "count": 2, + "mean": 5.891583859920502 + }, + "retrieval-only::embedding-bge-large::minilm": { + "wins": 0, + "losses": 0, + "count": 2, + "mean": 29.316066298633814 + }, + "retrieval-only::embedding-bge-large::mxbai": { + "wins": 0, + "losses": 0, + "count": 2, + "mean": 204.86171124503016 + }, + "retrieval-only::hybrid::none": { + "wins": 0, + "losses": 0, + "count": 2, + "mean": 6.42356975004077 + }, + "retrieval-only::hybrid::minilm": { + "wins": 0, + "losses": 0, + "count": 2, + "mean": 31.444795429706573 + }, + "retrieval-only::hybrid::mxbai": { + "wins": 0, + "losses": 0, + "count": 2, + "mean": 207.23473047837615 + }, + "retrieval-only::hybrid-bge-large::none": { + "wins": 0, + "losses": 0, + "count": 2, + "mean": 6.400614511221647 + }, + "retrieval-only::hybrid-bge-large::minilm": { + "wins": 0, + "losses": 0, + "count": 2, + "mean": 30.800541397184134 + }, + "retrieval-only::hybrid-bge-large::mxbai": { + "wins": 0, + "losses": 0, + "count": 2, + "mean": 206.9518780335784 + }, + "retrieval-only::splade-only::none": { + "wins": 0, + "losses": 0, + "count": 2, + "mean": 2.768915146589279 + }, + "retrieval-only::splade-only::minilm": { + "wins": 0, + "losses": 0, + "count": 2, + "mean": 26.865416672080755 + }, + "retrieval-only::splade-only::mxbai": { + "wins": 0, + "losses": 0, + "count": 2, + "mean": 202.86574540659785 + }, + "retrieval-only::hybrid-triple::none": { + "wins": 0, + "losses": 0, + "count": 2, + "mean": 9.532728232443333 + }, + "retrieval-only::hybrid-triple::minilm": { + "wins": 0, + "losses": 0, + "count": 2, + "mean": 34.74805224686861 + }, + "retrieval-only::hybrid-triple::mxbai": { + "wins": 0, + "losses": 0, + "count": 2, + "mean": 208.22617365047336 + }, + "retrieval-only::hybrid-web::none": { + "wins": 0, + "losses": 0, + "count": 2, + "mean": 2549.0026334300637 + }, + "retrieval-only::hybrid-web::minilm": { + "wins": 0, + "losses": 0, + "count": 2, + "mean": 5065.05661085248 + }, + "retrieval-only::hybrid-web::mxbai": { + "wins": 0, + "losses": 2, + "count": 2, + "mean": 5197.566072456539 + }, + "devstral-123b::hybrid::none": { + "wins": 0, + "losses": 5, + "count": 7, + "mean": 8.211523560541016 + }, + "ministral-14b-reasoning::hybrid::none": { + "wins": 1, + "losses": 0, + "count": 7, + "mean": 7.412772226546492 + }, + "magistral-small::hybrid::none": { + "wins": 0, + "losses": 0, + "count": 7, + "mean": 7.392910974366324 + }, + "devstral-small-24b::hybrid::none": { + "wins": 4, + "losses": 0, + "count": 7, + "mean": 7.296505916331496 + }, + "mistral-small-3.2-24b::hybrid::none": { + "wins": 0, + "losses": 0, + "count": 7, + "mean": 7.394192075090749 + }, + "granite-4-30b::hybrid::none": { + "wins": 0, + "losses": 0, + "count": 7, + "mean": 7.358867808112076 + } + }, + "p95": { + "retrieval-only::bm25-only::none": { + "wins": 2, + "losses": 0, + "count": 2, + "mean": 0.5216426216065884 + }, + "retrieval-only::bm25-only::minilm": { + "wins": 0, + "losses": 0, + "count": 2, + "mean": 24.78122152388096 + }, + "retrieval-only::bm25-only::mxbai": { + "wins": 0, + "losses": 0, + "count": 2, + "mean": 194.22026490792632 + }, + "retrieval-only::embedding-bge-m3::none": { + "wins": 0, + "losses": 0, + "count": 2, + "mean": 6.5811071544885635 + }, + "retrieval-only::embedding-bge-m3::minilm": { + "wins": 0, + "losses": 0, + "count": 2, + "mean": 32.704496290534735 + }, + "retrieval-only::embedding-bge-m3::mxbai": { + "wins": 0, + "losses": 0, + "count": 2, + "mean": 207.5708662159741 + }, + "retrieval-only::embedding-bge-large::none": { + "wins": 0, + "losses": 0, + "count": 2, + "mean": 6.531184073537588 + }, + "retrieval-only::embedding-bge-large::minilm": { + "wins": 0, + "losses": 0, + "count": 2, + "mean": 31.900114379823208 + }, + "retrieval-only::embedding-bge-large::mxbai": { + "wins": 0, + "losses": 0, + "count": 2, + "mean": 208.62957322970033 + }, + "retrieval-only::hybrid::none": { + "wins": 0, + "losses": 0, + "count": 2, + "mean": 7.1006291545927525 + }, + "retrieval-only::hybrid::minilm": { + "wins": 0, + "losses": 0, + "count": 2, + "mean": 33.13743881881237 + }, + "retrieval-only::hybrid::mxbai": { + "wins": 0, + "losses": 0, + "count": 2, + "mean": 209.9398672580719 + }, + "retrieval-only::hybrid-bge-large::none": { + "wins": 0, + "losses": 0, + "count": 2, + "mean": 7.356016896665096 + }, + "retrieval-only::hybrid-bge-large::minilm": { + "wins": 0, + "losses": 0, + "count": 2, + "mean": 32.61782508343458 + }, + "retrieval-only::hybrid-bge-large::mxbai": { + "wins": 0, + "losses": 0, + "count": 2, + "mean": 210.08411142975092 + }, + "retrieval-only::splade-only::none": { + "wins": 0, + "losses": 0, + "count": 2, + "mean": 3.0150958336889744 + }, + "retrieval-only::splade-only::minilm": { + "wins": 0, + "losses": 0, + "count": 2, + "mean": 28.595024719834328 + }, + "retrieval-only::splade-only::mxbai": { + "wins": 0, + "losses": 0, + "count": 2, + "mean": 206.21620258316398 + }, + "retrieval-only::hybrid-triple::none": { + "wins": 0, + "losses": 0, + "count": 2, + "mean": 10.518142487853765 + }, + "retrieval-only::hybrid-triple::minilm": { + "wins": 0, + "losses": 0, + "count": 2, + "mean": 37.638560868799686 + }, + "retrieval-only::hybrid-triple::mxbai": { + "wins": 0, + "losses": 0, + "count": 2, + "mean": 211.2135230563581 + }, + "retrieval-only::hybrid-web::none": { + "wins": 0, + "losses": 0, + "count": 2, + "mean": 5030.530819203705 + }, + "retrieval-only::hybrid-web::minilm": { + "wins": 0, + "losses": 0, + "count": 2, + "mean": 5073.805775493383 + }, + "retrieval-only::hybrid-web::mxbai": { + "wins": 0, + "losses": 2, + "count": 2, + "mean": 5225.449657533318 + }, + "devstral-123b::hybrid::none": { + "wins": 0, + "losses": 5, + "count": 7, + "mean": 9.212801099887916 + }, + "ministral-14b-reasoning::hybrid::none": { + "wins": 0, + "losses": 0, + "count": 7, + "mean": 7.984842838985579 + }, + "magistral-small::hybrid::none": { + "wins": 1, + "losses": 0, + "count": 7, + "mean": 8.029126961316381 + }, + "devstral-small-24b::hybrid::none": { + "wins": 2, + "losses": 0, + "count": 7, + "mean": 7.915892239127841 + }, + "mistral-small-3.2-24b::hybrid::none": { + "wins": 0, + "losses": 0, + "count": 7, + "mean": 7.930300065449306 + }, + "granite-4-30b::hybrid::none": { + "wins": 2, + "losses": 0, + "count": 7, + "mean": 7.880111091903278 + } + }, + "build": { + "retrieval-only::bm25-only::none": { + "wins": 0, + "losses": 0, + "count": 2, + "mean": 0.6894275164231658 + }, + "retrieval-only::bm25-only::minilm": { + "wins": 0, + "losses": 0, + "count": 2, + "mean": 0.010334129445254803 + }, + "retrieval-only::bm25-only::mxbai": { + "wins": 0, + "losses": 0, + "count": 2, + "mean": 0.01179773872718215 + }, + "retrieval-only::embedding-bge-m3::none": { + "wins": 0, + "losses": 0, + "count": 2, + "mean": 0.02554283756762743 + }, + "retrieval-only::embedding-bge-m3::minilm": { + "wins": 0, + "losses": 0, + "count": 2, + "mean": 0.011171562131494284 + }, + "retrieval-only::embedding-bge-m3::mxbai": { + "wins": 0, + "losses": 0, + "count": 2, + "mean": 0.02657144283875823 + }, + "retrieval-only::embedding-bge-large::none": { + "wins": 0, + "losses": 0, + "count": 2, + "mean": 0.010330421384423971 + }, + "retrieval-only::embedding-bge-large::minilm": { + "wins": 0, + "losses": 0, + "count": 2, + "mean": 0.010938801802694798 + }, + "retrieval-only::embedding-bge-large::mxbai": { + "wins": 0, + "losses": 0, + "count": 2, + "mean": 0.011622074991464615 + }, + "retrieval-only::hybrid::none": { + "wins": 0, + "losses": 0, + "count": 2, + "mean": 0.032247269526124 + }, + "retrieval-only::hybrid::minilm": { + "wins": 0, + "losses": 0, + "count": 2, + "mean": 0.016082720831036568 + }, + "retrieval-only::hybrid::mxbai": { + "wins": 0, + "losses": 0, + "count": 2, + "mean": 0.03033801121637225 + }, + "retrieval-only::hybrid-bge-large::none": { + "wins": 0, + "losses": 0, + "count": 2, + "mean": 0.016932107508182526 + }, + "retrieval-only::hybrid-bge-large::minilm": { + "wins": 0, + "losses": 0, + "count": 2, + "mean": 0.016773953568190336 + }, + "retrieval-only::hybrid-bge-large::mxbai": { + "wins": 0, + "losses": 0, + "count": 2, + "mean": 0.016854459885507822 + }, + "retrieval-only::splade-only::none": { + "wins": 0, + "losses": 0, + "count": 2, + "mean": 0.007179258391261101 + }, + "retrieval-only::splade-only::minilm": { + "wins": 1, + "losses": 0, + "count": 2, + "mean": 0.006352534517645836 + }, + "retrieval-only::splade-only::mxbai": { + "wins": 1, + "losses": 0, + "count": 2, + "mean": 0.01976611651480198 + }, + "retrieval-only::hybrid-triple::none": { + "wins": 0, + "losses": 0, + "count": 2, + "mean": 0.018045139964669943 + }, + "retrieval-only::hybrid-triple::minilm": { + "wins": 0, + "losses": 0, + "count": 2, + "mean": 0.018401450011879206 + }, + "retrieval-only::hybrid-triple::mxbai": { + "wins": 0, + "losses": 0, + "count": 2, + "mean": 0.018003772478550673 + }, + "retrieval-only::hybrid-web::none": { + "wins": 0, + "losses": 2, + "count": 2, + "mean": 15.669111443683505 + }, + "retrieval-only::hybrid-web::minilm": { + "wins": 0, + "losses": 0, + "count": 2, + "mean": 14.033660519868135 + }, + "retrieval-only::hybrid-web::mxbai": { + "wins": 0, + "losses": 0, + "count": 2, + "mean": 13.982121753040701 + }, + "devstral-123b::hybrid::none": { + "wins": 2, + "losses": 2, + "count": 7, + "mean": 0.033393271932644505 + }, + "ministral-14b-reasoning::hybrid::none": { + "wins": 2, + "losses": 1, + "count": 7, + "mean": 0.03156101557293108 + }, + "magistral-small::hybrid::none": { + "wins": 1, + "losses": 0, + "count": 7, + "mean": 0.02820786434624876 + }, + "devstral-small-24b::hybrid::none": { + "wins": 0, + "losses": 0, + "count": 7, + "mean": 0.027854623539107188 + }, + "mistral-small-3.2-24b::hybrid::none": { + "wins": 0, + "losses": 0, + "count": 7, + "mean": 0.028560412781579152 + }, + "granite-4-30b::hybrid::none": { + "wins": 0, + "losses": 2, + "count": 7, + "mean": 0.02849655132740736 + } + }, + "gen": { + "devstral-123b::hybrid::none": { + "wins": 7, + "losses": 0, + "count": 7, + "mean": 0.7942857142857144 + } + }, + "gen_ms": { + "devstral-123b::hybrid::none": { + "wins": 7, + "losses": 0, + "count": 7, + "mean": 582.9223734991891 + } + }, + "tok_s": { + "devstral-123b::hybrid::none": { + "wins": 7, + "losses": 0, + "count": 7, + "mean": 636.8282663437029 + } + }, + "gen_cost": { + "devstral-123b::hybrid::none": { + "wins": 7, + "losses": 0, + "count": 7, + "mean": 0.0009117746938775508 + } + }, + "cost": { + "retrieval-only::bm25-only::none": { + "wins": 2, + "losses": 0, + "count": 2, + "mean": 0.0 + }, + "retrieval-only::bm25-only::minilm": { + "wins": 2, + "losses": 0, + "count": 2, + "mean": 0.0 + }, + "retrieval-only::bm25-only::mxbai": { + "wins": 2, + "losses": 0, + "count": 2, + "mean": 0.0 + }, + "retrieval-only::embedding-bge-m3::none": { + "wins": 2, + "losses": 0, + "count": 2, + "mean": 0.0 + }, + "retrieval-only::embedding-bge-m3::minilm": { + "wins": 2, + "losses": 0, + "count": 2, + "mean": 0.0 + }, + "retrieval-only::embedding-bge-m3::mxbai": { + "wins": 2, + "losses": 0, + "count": 2, + "mean": 0.0 + }, + "retrieval-only::embedding-bge-large::none": { + "wins": 2, + "losses": 0, + "count": 2, + "mean": 0.0 + }, + "retrieval-only::embedding-bge-large::minilm": { + "wins": 2, + "losses": 0, + "count": 2, + "mean": 0.0 + }, + "retrieval-only::embedding-bge-large::mxbai": { + "wins": 2, + "losses": 0, + "count": 2, + "mean": 0.0 + }, + "retrieval-only::hybrid::none": { + "wins": 2, + "losses": 0, + "count": 2, + "mean": 0.0 + }, + "retrieval-only::hybrid::minilm": { + "wins": 2, + "losses": 0, + "count": 2, + "mean": 0.0 + }, + "retrieval-only::hybrid::mxbai": { + "wins": 2, + "losses": 0, + "count": 2, + "mean": 0.0 + }, + "retrieval-only::hybrid-bge-large::none": { + "wins": 2, + "losses": 0, + "count": 2, + "mean": 0.0 + }, + "retrieval-only::hybrid-bge-large::minilm": { + "wins": 2, + "losses": 0, + "count": 2, + "mean": 0.0 + }, + "retrieval-only::hybrid-bge-large::mxbai": { + "wins": 2, + "losses": 0, + "count": 2, + "mean": 0.0 + }, + "retrieval-only::splade-only::none": { + "wins": 2, + "losses": 0, + "count": 2, + "mean": 0.0 + }, + "retrieval-only::splade-only::minilm": { + "wins": 2, + "losses": 0, + "count": 2, + "mean": 0.0 + }, + "retrieval-only::splade-only::mxbai": { + "wins": 2, + "losses": 0, + "count": 2, + "mean": 0.0 + }, + "retrieval-only::hybrid-triple::none": { + "wins": 2, + "losses": 0, + "count": 2, + "mean": 0.0 + }, + "retrieval-only::hybrid-triple::minilm": { + "wins": 2, + "losses": 0, + "count": 2, + "mean": 0.0 + }, + "retrieval-only::hybrid-triple::mxbai": { + "wins": 2, + "losses": 0, + "count": 2, + "mean": 0.0 + }, + "retrieval-only::hybrid-web::none": { + "wins": 2, + "losses": 0, + "count": 2, + "mean": 0.0 + }, + "retrieval-only::hybrid-web::minilm": { + "wins": 2, + "losses": 0, + "count": 2, + "mean": 0.0 + }, + "retrieval-only::hybrid-web::mxbai": { + "wins": 2, + "losses": 0, + "count": 2, + "mean": 0.0 + }, + "devstral-123b::hybrid::none": { + "wins": 0, + "losses": 7, + "count": 7, + "mean": 0.0009117746938775508 + }, + "ministral-14b-reasoning::hybrid::none": { + "wins": 7, + "losses": 0, + "count": 7, + "mean": 0.0 + }, + "magistral-small::hybrid::none": { + "wins": 7, + "losses": 0, + "count": 7, + "mean": 0.0 + }, + "devstral-small-24b::hybrid::none": { + "wins": 7, + "losses": 0, + "count": 7, + "mean": 0.0 + }, + "mistral-small-3.2-24b::hybrid::none": { + "wins": 7, + "losses": 0, + "count": 7, + "mean": 0.0 + }, + "granite-4-30b::hybrid::none": { + "wins": 7, + "losses": 0, + "count": 7, + "mean": 0.0 + } + }, + "par": { + "retrieval-only::bm25-only::none": { + "wins": 0, + "losses": 0, + "count": 2, + "mean": 0.03598809523809535 + }, + "retrieval-only::bm25-only::minilm": { + "wins": 0, + "losses": 0, + "count": 2, + "mean": 0.02162698412698408 + }, + "retrieval-only::bm25-only::mxbai": { + "wins": 0, + "losses": 1, + "count": 2, + "mean": 0.02560119047619044 + }, + "retrieval-only::embedding-bge-m3::none": { + "wins": 0, + "losses": 0, + "count": 2, + "mean": 0.05380158730158733 + }, + "retrieval-only::embedding-bge-m3::minilm": { + "wins": 0, + "losses": 0, + "count": 2, + "mean": 0.022214285714285728 + }, + "retrieval-only::embedding-bge-m3::mxbai": { + "wins": 0, + "losses": 0, + "count": 2, + "mean": 0.05740873015873019 + }, + "retrieval-only::embedding-bge-large::none": { + "wins": 0, + "losses": 0, + "count": 2, + "mean": 0.04175396825396826 + }, + "retrieval-only::embedding-bge-large::minilm": { + "wins": 0, + "losses": 0, + "count": 2, + "mean": 0.01264880952380959 + }, + "retrieval-only::embedding-bge-large::mxbai": { + "wins": 1, + "losses": 0, + "count": 2, + "mean": 0.05530555555555562 + }, + "retrieval-only::hybrid::none": { + "wins": 0, + "losses": 0, + "count": 2, + "mean": 0.04750595238095236 + }, + "retrieval-only::hybrid::minilm": { + "wins": 0, + "losses": 0, + "count": 2, + "mean": 0.017311507936507882 + }, + "retrieval-only::hybrid::mxbai": { + "wins": 0, + "losses": 0, + "count": 2, + "mean": 0.06067063492063492 + }, + "retrieval-only::hybrid-bge-large::none": { + "wins": 0, + "losses": 0, + "count": 2, + "mean": 0.03945833333333328 + }, + "retrieval-only::hybrid-bge-large::minilm": { + "wins": 1, + "losses": 0, + "count": 2, + "mean": 0.03327380952380961 + }, + "retrieval-only::hybrid-bge-large::mxbai": { + "wins": 0, + "losses": 0, + "count": 2, + "mean": 0.06022619047619054 + }, + "retrieval-only::splade-only::none": { + "wins": 0, + "losses": 1, + "count": 2, + "mean": 0.002821428571428624 + }, + "retrieval-only::splade-only::minilm": { + "wins": 0, + "losses": 0, + "count": 2, + "mean": 0.016950396825396827 + }, + "retrieval-only::splade-only::mxbai": { + "wins": 0, + "losses": 0, + "count": 2, + "mean": 0.04500198412698413 + }, + "retrieval-only::hybrid-triple::none": { + "wins": 0, + "losses": 0, + "count": 2, + "mean": 0.050505952380952374 + }, + "retrieval-only::hybrid-triple::minilm": { + "wins": 0, + "losses": 0, + "count": 2, + "mean": 0.015744047619047637 + }, + "retrieval-only::hybrid-triple::mxbai": { + "wins": 0, + "losses": 0, + "count": 2, + "mean": 0.052242063492063576 + }, + "retrieval-only::hybrid-web::none": { + "wins": 0, + "losses": 0, + "count": 2, + "mean": 0.04750595238095236 + }, + "retrieval-only::hybrid-web::minilm": { + "wins": 0, + "losses": 0, + "count": 2, + "mean": 0.017311507936507882 + }, + "retrieval-only::hybrid-web::mxbai": { + "wins": 0, + "losses": 0, + "count": 2, + "mean": 0.06067063492063492 + }, + "devstral-123b::hybrid::none": { + "wins": 5, + "losses": 0, + "count": 7, + "mean": 0.028471088435374135 + }, + "ministral-14b-reasoning::hybrid::none": { + "wins": 5, + "losses": 0, + "count": 7, + "mean": 0.028471088435374135 + }, + "magistral-small::hybrid::none": { + "wins": 5, + "losses": 0, + "count": 7, + "mean": 0.028471088435374135 + }, + "devstral-small-24b::hybrid::none": { + "wins": 5, + "losses": 0, + "count": 7, + "mean": 0.028471088435374135 + }, + "mistral-small-3.2-24b::hybrid::none": { + "wins": 5, + "losses": 0, + "count": 7, + "mean": 0.028471088435374135 + }, + "granite-4-30b::hybrid::none": { + "wins": 5, + "losses": 0, + "count": 7, + "mean": 0.028471088435374135 + } + }, + "shift": { + "retrieval-only::bm25-only::none": { + "wins": 0, + "losses": 1, + "count": 2, + "mean": 0.067654761904762 + }, + "retrieval-only::bm25-only::minilm": { + "wins": 0, + "losses": 0, + "count": 2, + "mean": 0.15779166666666675 + }, + "retrieval-only::bm25-only::mxbai": { + "wins": 0, + "losses": 0, + "count": 2, + "mean": 0.10380753968253964 + }, + "retrieval-only::embedding-bge-m3::none": { + "wins": 0, + "losses": 1, + "count": 2, + "mean": 0.06060119047619052 + }, + "retrieval-only::embedding-bge-m3::minilm": { + "wins": 0, + "losses": 0, + "count": 2, + "mean": 0.14105753968253976 + }, + "retrieval-only::embedding-bge-m3::mxbai": { + "wins": 0, + "losses": 0, + "count": 2, + "mean": 0.11650198412698412 + }, + "retrieval-only::embedding-bge-large::none": { + "wins": 0, + "losses": 0, + "count": 2, + "mean": 0.09962301587301593 + }, + "retrieval-only::embedding-bge-large::minilm": { + "wins": 0, + "losses": 0, + "count": 2, + "mean": 0.1678293650793652 + }, + "retrieval-only::embedding-bge-large::mxbai": { + "wins": 0, + "losses": 0, + "count": 2, + "mean": 0.10281349206349212 + }, + "retrieval-only::hybrid::none": { + "wins": 0, + "losses": 0, + "count": 2, + "mean": 0.17769047619047618 + }, + "retrieval-only::hybrid::minilm": { + "wins": 0, + "losses": 0, + "count": 2, + "mean": 0.15861309523809525 + }, + "retrieval-only::hybrid::mxbai": { + "wins": 0, + "losses": 0, + "count": 2, + "mean": 0.11200992063492066 + }, + "retrieval-only::hybrid-bge-large::none": { + "wins": 0, + "losses": 0, + "count": 2, + "mean": 0.12747023809523814 + }, + "retrieval-only::hybrid-bge-large::minilm": { + "wins": 0, + "losses": 0, + "count": 2, + "mean": 0.16234948979591846 + }, + "retrieval-only::hybrid-bge-large::mxbai": { + "wins": 0, + "losses": 0, + "count": 2, + "mean": 0.09213945578231302 + }, + "retrieval-only::splade-only::none": { + "wins": 0, + "losses": 0, + "count": 2, + "mean": 0.16295238095238102 + }, + "retrieval-only::splade-only::minilm": { + "wins": 1, + "losses": 0, + "count": 2, + "mean": 0.19781944444444455 + }, + "retrieval-only::splade-only::mxbai": { + "wins": 0, + "losses": 0, + "count": 2, + "mean": 0.08979761904761907 + }, + "retrieval-only::hybrid-triple::none": { + "wins": 0, + "losses": 0, + "count": 2, + "mean": 0.16739880952380956 + }, + "retrieval-only::hybrid-triple::minilm": { + "wins": 1, + "losses": 0, + "count": 2, + "mean": 0.18809523809523815 + }, + "retrieval-only::hybrid-triple::mxbai": { + "wins": 0, + "losses": 0, + "count": 2, + "mean": 0.11244841269841283 + }, + "retrieval-only::hybrid-web::none": { + "wins": 0, + "losses": 0, + "count": 2, + "mean": 0.17769047619047618 + }, + "retrieval-only::hybrid-web::minilm": { + "wins": 0, + "losses": 0, + "count": 2, + "mean": 0.15861309523809525 + }, + "retrieval-only::hybrid-web::mxbai": { + "wins": 0, + "losses": 0, + "count": 2, + "mean": 0.11200992063492066 + }, + "devstral-123b::hybrid::none": { + "wins": 5, + "losses": 0, + "count": 7, + "mean": 0.2000913508260447 + }, + "ministral-14b-reasoning::hybrid::none": { + "wins": 5, + "losses": 0, + "count": 7, + "mean": 0.2000913508260447 + }, + "magistral-small::hybrid::none": { + "wins": 5, + "losses": 0, + "count": 7, + "mean": 0.2000913508260447 + }, + "devstral-small-24b::hybrid::none": { + "wins": 5, + "losses": 0, + "count": 7, + "mean": 0.2000913508260447 + }, + "mistral-small-3.2-24b::hybrid::none": { + "wins": 5, + "losses": 0, + "count": 7, + "mean": 0.2000913508260447 + }, + "granite-4-30b::hybrid::none": { + "wins": 5, + "losses": 0, + "count": 7, + "mean": 0.2000913508260447 + } + } + }, + "metrics_def": [ + { + "key": "mrr", + "label": "MRR", + "field": "mrr", + "higher": true, + "fmt": "f3", + "tip": "MRR" + }, + { + "key": "h1", + "label": "H@1", + "field": "hit_at_1", + "higher": true, + "fmt": "f2", + "tip": "H@1" + }, + { + "key": "h5", + "label": "H@5", + "field": "hit_at_5", + "higher": true, + "fmt": "f2", + "tip": "H@5" + }, + { + "key": "ndcg", + "label": "nDCG", + "field": "ndcg_at_10", + "higher": true, + "fmt": "f3", + "tip": "nDCG" + }, + { + "key": "p50", + "label": "P50 ms", + "field": "p50_retrieval_latency_ms", + "higher": false, + "fmt": "f0", + "tip": "P50" + }, + { + "key": "p95", + "label": "P95 ms", + "field": "p95_retrieval_latency_ms", + "higher": false, + "fmt": "f0", + "tip": "P95" + }, + { + "key": "build", + "label": "Build s", + "field": "index_build_seconds", + "higher": false, + "fmt": "f1", + "tip": "build" + }, + { + "key": "gen", + "label": "gen", + "field": "generation_acc", + "higher": true, + "fmt": "f3", + "tip": "gen" + }, + { + "key": "gen_ms", + "label": "gen ms", + "field": "p50_generation_latency_ms", + "higher": false, + "fmt": "f0", + "tip": "gen ms" + }, + { + "key": "tok_s", + "label": "tok/s", + "field": "generation_tokens_per_sec", + "higher": true, + "fmt": "f0", + "tip": "tok/s" + }, + { + "key": "gen_cost", + "label": "gen $", + "field": "est_generation_cost_per_query_usd", + "higher": false, + "fmt": "f5", + "tip": "gen $" + }, + { + "key": "cost", + "label": "$/q", + "field": "est_cost_per_query_usd", + "higher": false, + "fmt": "f5", + "tip": "$/q" + }, + { + "key": "par", + "label": "par", + "field": "paraphrase_drop", + "higher": true, + "fmt": "f3", + "tip": "par" + }, + { + "key": "shift", + "label": "shift", + "field": "shift_drop", + "higher": true, + "fmt": "f3", + "tip": "shift" + } + ], + "glossary": { + "model": "The generation model in the pipeline. 'retrieval-only' means no LLM generation step was run for this row.", + "retriever": "Retrieval strategy used to fetch candidate documents (e.g. hybrid = BM25 + dense embeddings; bm25-only = lexical; splade-only = learned sparse).", + "rerank": "Cross-encoder reranker applied after retrieval ('none' = no reranking; minilm / mxbai are reranker models).", + "MRR": "Mean Reciprocal Rank — average of 1/rank for the first relevant doc retrieved per query. Range 0–1; higher is better.", + "H@1": "Hit@1 — fraction of queries where the top-1 retrieved doc is relevant. Higher is better.", + "H@5": "Hit@5 — fraction of queries where any of the top-5 retrieved docs is relevant. Higher is better.", + "nDCG": "Normalized Discounted Cumulative Gain @10 — position-weighted relevance of the top-10 results. Higher is better.", + "P50": "Median (50th percentile) per-query retrieval latency in milliseconds. Lower is better.", + "P95": "95th percentile retrieval latency — the tail (worst-case) cost per query, in milliseconds. Lower is better.", + "build": "Index build time in seconds for this benchmark corpus. Lower is better.", + "gen": "Generation score (task-dependent): for multiple-choice benchmarks, the fraction of questions answered correctly given retrieved context; for the language-following benchmark, the fraction of responses written in the user's language. Higher is better.", + "gen ms": "Median per-query generation latency (time-to-first-token + generation) in milliseconds. Lower is better.", + "tok/s": "LLM output tokens generated per second on this benchmark. Higher is better.", + "gen $": "Estimated generation cost per query in USD (token usage × model price). Lower is better.", + "$/q": "Total estimated cost per query in USD (retrieval + generation). Lower is better.", + "par": "Paraphrase robustness — MRR on paraphrased queries minus MRR on originals. Near zero is expected; large negative means fragile exact-match retrieval.", + "shift": "Held-out (domain-shifted) slice MRR — measures out-of-distribution generalization on an unseen query subset.", + "n": "Number of queries evaluated for this row.", + "verdict": "Whether the winning candidate is statistically distinct from the runner-up. 'winner clear' = winner's Wilson CI lower bound exceeds runner-up's upper bound; 'inconclusive' = confidence intervals overlap (within noise); 'no signal' = too few valid candidates; 'all disqualified' = every candidate failed a hard gate.", + "Pareto": "Pareto front — a pipeline not dominated by any other on (accuracy, latency, build-cost) simultaneously. Marked with ★.", + "DQ": "Disqualified — violated one of the profile's hard gates: DQ:latency (p50 over budget), DQ:cost (cost per query over ceiling), or DQ:accuracy (MRR or generation score below the floor).", + "pick": "The router's single recommended pipeline (model / retriever / reranker) for this client profile on this benchmark.", + "profile": "A client requirements file (not a model) declaring latency / cost / accuracy hard gates plus utility weights used to rank surviving candidates." + }, + "provenance": { + "merged_from": [ + "reports/weekly/2026-06-29/retrieval/raw/run.json", + "reports/weekly/2026-06-29/models/devstral-123b/raw/run.json", + "reports/weekly/2026-06-29/models/ministral-14b-reasoning/raw/run.json", + "reports/weekly/2026-06-29/models/magistral-small/raw/run.json", + "reports/weekly/2026-06-29/models/devstral-small-24b/raw/run.json", + "reports/weekly/2026-06-29/models/mistral-small-3.2-24b/raw/run.json", + "reports/weekly/2026-06-29/models/granite-4-30b/raw/run.json" + ], + "merged_at": "2026-06-29T15:28:40.311551+00:00", + "weekly": true + }, + "n_candidates": 30, + "n_queries": 100 +} \ No newline at end of file