{ "model_dir": "/train-runs/clerk-sft-v1/merged", "device": "cuda:0", "temperature": 0.0, "overall": { "n": 40, "skipped": 0, "accuracy": 1.0, "median_s": 0.061, "tokens": 176 }, "by_role": { "rag": { "n": 40, "skipped": 0, "accuracy": 1.0, "median_s": 0.061, "tokens": 176 } }, "rows": [ { "role": "rag", "task_id": "rag-001", "ok": true, "answer": "card1", "wall_s": 0.573, "tokens": 3, "error": null }, { "role": "rag", "task_id": "rag-002", "ok": true, "answer": "32k", "wall_s": 0.052, "tokens": 3, "error": null }, { "role": "rag", "task_id": "rag-003", "ok": true, "answer": "card0, 32k", "wall_s": 0.115, "tokens": 7, "error": null }, { "role": "rag", "task_id": "rag-004", "ok": true, "answer": "9.54", "wall_s": 0.068, "tokens": 4, "error": null }, { "role": "rag", "task_id": "rag-204", "ok": true, "answer": "card 0, 32k", "wall_s": 0.133, "tokens": 8, "error": null }, { "role": "rag", "task_id": "rag-205", "ok": true, "answer": "2", "wall_s": 0.038, "tokens": 2, "error": null }, { "role": "rag", "task_id": "rag-206", "ok": true, "answer": "2", "wall_s": 0.038, "tokens": 2, "error": null }, { "role": "rag", "task_id": "rag-207", "ok": true, "answer": "1", "wall_s": 0.038, "tokens": 2, "error": null }, { "role": "rag", "task_id": "rag-208", "ok": true, "answer": "card 1, 8k", "wall_s": 0.132, "tokens": 8, "error": null }, { "role": "rag", "task_id": "rag-209", "ok": true, "answer": "card 1, 8k", "wall_s": 0.133, "tokens": 8, "error": null }, { "role": "rag", "task_id": "rag-210", "ok": true, "answer": "9.54", "wall_s": 0.07, "tokens": 4, "error": null }, { "role": "rag", "task_id": "rag-211", "ok": true, "answer": "4.99", "wall_s": 0.07, "tokens": 4, "error": null }, { "role": "rag", "task_id": "rag-212", "ok": true, "answer": "1", "wall_s": 0.038, "tokens": 2, "error": null }, { "role": "rag", "task_id": "rag-213", "ok": true, "answer": "2", "wall_s": 0.038, "tokens": 2, "error": null }, { "role": "rag", "task_id": "rag-214", "ok": true, "answer": "card 0, 16k", "wall_s": 0.133, "tokens": 8, "error": null }, { "role": "rag", "task_id": "rag-215", "ok": true, "answer": "card 0, 32k", "wall_s": 0.133, "tokens": 8, "error": null }, { "role": "rag", "task_id": "rag-216", "ok": true, "answer": "9.98", "wall_s": 0.069, "tokens": 4, "error": null }, { "role": "rag", "task_id": "rag-217", "ok": true, "answer": "16k", "wall_s": 0.054, "tokens": 3, "error": null }, { "role": "rag", "task_id": "rag-218", "ok": true, "answer": "1", "wall_s": 0.038, "tokens": 2, "error": null }, { "role": "rag", "task_id": "rag-219", "ok": true, "answer": "card 0, 16k", "wall_s": 0.133, "tokens": 8, "error": null }, { "role": "rag", "task_id": "rag-220", "ok": true, "answer": "16k", "wall_s": 0.054, "tokens": 3, "error": null }, { "role": "rag", "task_id": "rag-221", "ok": true, "answer": "card 1, 8k", "wall_s": 0.132, "tokens": 8, "error": null }, { "role": "rag", "task_id": "rag-222", "ok": true, "answer": "2", "wall_s": 0.038, "tokens": 2, "error": null }, { "role": "rag", "task_id": "rag-223", "ok": true, "answer": "8k", "wall_s": 0.054, "tokens": 3, "error": null }, { "role": "rag", "task_id": "rag-224", "ok": true, "answer": "8k", "wall_s": 0.054, "tokens": 3, "error": null }, { "role": "rag", "task_id": "rag-225", "ok": true, "answer": "9.54", "wall_s": 0.07, "tokens": 4, "error": null }, { "role": "rag", "task_id": "rag-226", "ok": true, "answer": "32k", "wall_s": 0.054, "tokens": 3, "error": null }, { "role": "rag", "task_id": "rag-227", "ok": true, "answer": "card 1, 64k", "wall_s": 0.133, "tokens": 8, "error": null }, { "role": "rag", "task_id": "rag-228", "ok": true, "answer": "card 1, 8k", "wall_s": 0.133, "tokens": 8, "error": null }, { "role": "rag", "task_id": "rag-229", "ok": true, "answer": "32k", "wall_s": 0.054, "tokens": 3, "error": null }, { "role": "rag", "task_id": "rag-230", "ok": true, "answer": "0", "wall_s": 0.038, "tokens": 2, "error": null }, { "role": "rag", "task_id": "rag-231", "ok": true, "answer": "8k", "wall_s": 0.054, "tokens": 3, "error": null }, { "role": "rag", "task_id": "rag-232", "ok": true, "answer": "5.69", "wall_s": 0.07, "tokens": 4, "error": null }, { "role": "rag", "task_id": "rag-233", "ok": true, "answer": "4.99", "wall_s": 0.07, "tokens": 4, "error": null }, { "role": "rag", "task_id": "rag-234", "ok": true, "answer": "card 1, 8k", "wall_s": 0.132, "tokens": 8, "error": null }, { "role": "rag", "task_id": "rag-235", "ok": true, "answer": "2", "wall_s": 0.038, "tokens": 2, "error": null }, { "role": "rag", "task_id": "rag-236", "ok": true, "answer": "32k", "wall_s": 0.054, "tokens": 3, "error": null }, { "role": "rag", "task_id": "rag-237", "ok": true, "answer": "8k", "wall_s": 0.054, "tokens": 3, "error": null }, { "role": "rag", "task_id": "rag-238", "ok": true, "answer": "card 0, 32k", "wall_s": 0.133, "tokens": 8, "error": null }, { "role": "rag", "task_id": "rag-239", "ok": true, "answer": "2", "wall_s": 0.038, "tokens": 2, "error": null } ] }