| { |
| "status": "passed", |
| "date": "2026-10-04", |
| "execution": "server GPU; pre-existing cached backbone; network disabled", |
| "method": "QIME-TF-MMR", |
| "num_questions": 8855, |
| "topk": 256, |
| "mmr_lambda": 0.7, |
| "backbone_revision": "963121bfb9c625475f65b08fb54990ce9c4e7a1a", |
| "questions_sha256": "79b00c0675cdf38a0a5de0bbae7c055a0c3072bf8a868ae362d739fe85ea5e74", |
| "native_source_sha256": "b5916b6b6e724ca7663917229f8d3979700b23ad3690a18bf2bd2607aac30e1c", |
| "smoke_input_count": 6, |
| "output_shape": [ |
| 6, |
| 8855 |
| ], |
| "output_dtype": "float32", |
| "active_dimensions_per_text": [ |
| 256, |
| 256, |
| 256, |
| 256, |
| 256, |
| 256 |
| ], |
| "native_comparison": { |
| "same_backbone_and_question_tensors": true, |
| "binary_disagreements": 0, |
| "mmr_selection_order_equal": true |
| }, |
| "native_full_batch_selection_order_equal": true, |
| "chunked_batch_comparison": { |
| "batch_size": 2, |
| "binary_disagreements": 0 |
| }, |
| "synthetic_mmr_configurations_passed": 9, |
| "additional_checks": [ |
| "single-string input", |
| "empty input", |
| "invalid input validation", |
| "question-bank tamper detection" |
| ], |
| "versions": { |
| "torch": "2.6.0+cu124", |
| "sentence_transformers": "3.4.1", |
| "huggingface_hub": "0.36.0", |
| "numpy": "1.26.4" |
| }, |
| "paper_benchmark_reproduction": "reproduced", |
| "paper_benchmark_reproduction_details": "benchmark_reproduction.json", |
| "paper_benchmark_completed_tasks": 12, |
| "paper_benchmark_total_tasks": 12, |
| "full_twelve_task_reproduction_complete": true, |
| "paper_benchmark_author_reported_completed_tasks": [ |
| "MedrxivClusteringP2P", |
| "MedrxivClusteringS2S", |
| "ClusTREC-Covid" |
| ] |
| } |
|
|
