hive-reader-1.7b-v1 / eval /eval-rag.json
SkyIsNotGreen's picture
release: clean card, eval paths, hashes and adapter scope
3d831a3 verified
Raw History Blame Contribute Delete
6.01 kB
{
"model_dir": "<work>/train-runs/clerk-sft-v1/merged",
"device": "cuda:0",
"temperature": 0.0,
"overall": {
"n": 40,
"skipped": 0,
"accuracy": 1.0,
"median_s": 0.061,
"tokens": 176
},
"by_role": {
"rag": {
"n": 40,
"skipped": 0,
"accuracy": 1.0,
"median_s": 0.061,
"tokens": 176
}
},
"rows": [
{
"role": "rag",
"task_id": "rag-001",
"ok": true,
"answer": "card1",
"wall_s": 0.573,
"tokens": 3,
"error": null
},
{
"role": "rag",
"task_id": "rag-002",
"ok": true,
"answer": "32k",
"wall_s": 0.052,
"tokens": 3,
"error": null
},
{
"role": "rag",
"task_id": "rag-003",
"ok": true,
"answer": "card0, 32k",
"wall_s": 0.115,
"tokens": 7,
"error": null
},
{
"role": "rag",
"task_id": "rag-004",
"ok": true,
"answer": "9.54",
"wall_s": 0.068,
"tokens": 4,
"error": null
},
{
"role": "rag",
"task_id": "rag-204",
"ok": true,
"answer": "card 0, 32k",
"wall_s": 0.133,
"tokens": 8,
"error": null
},
{
"role": "rag",
"task_id": "rag-205",
"ok": true,
"answer": "2",
"wall_s": 0.038,
"tokens": 2,
"error": null
},
{
"role": "rag",
"task_id": "rag-206",
"ok": true,
"answer": "2",
"wall_s": 0.038,
"tokens": 2,
"error": null
},
{
"role": "rag",
"task_id": "rag-207",
"ok": true,
"answer": "1",
"wall_s": 0.038,
"tokens": 2,
"error": null
},
{
"role": "rag",
"task_id": "rag-208",
"ok": true,
"answer": "card 1, 8k",
"wall_s": 0.132,
"tokens": 8,
"error": null
},
{
"role": "rag",
"task_id": "rag-209",
"ok": true,
"answer": "card 1, 8k",
"wall_s": 0.133,
"tokens": 8,
"error": null
},
{
"role": "rag",
"task_id": "rag-210",
"ok": true,
"answer": "9.54",
"wall_s": 0.07,
"tokens": 4,
"error": null
},
{
"role": "rag",
"task_id": "rag-211",
"ok": true,
"answer": "4.99",
"wall_s": 0.07,
"tokens": 4,
"error": null
},
{
"role": "rag",
"task_id": "rag-212",
"ok": true,
"answer": "1",
"wall_s": 0.038,
"tokens": 2,
"error": null
},
{
"role": "rag",
"task_id": "rag-213",
"ok": true,
"answer": "2",
"wall_s": 0.038,
"tokens": 2,
"error": null
},
{
"role": "rag",
"task_id": "rag-214",
"ok": true,
"answer": "card 0, 16k",
"wall_s": 0.133,
"tokens": 8,
"error": null
},
{
"role": "rag",
"task_id": "rag-215",
"ok": true,
"answer": "card 0, 32k",
"wall_s": 0.133,
"tokens": 8,
"error": null
},
{
"role": "rag",
"task_id": "rag-216",
"ok": true,
"answer": "9.98",
"wall_s": 0.069,
"tokens": 4,
"error": null
},
{
"role": "rag",
"task_id": "rag-217",
"ok": true,
"answer": "16k",
"wall_s": 0.054,
"tokens": 3,
"error": null
},
{
"role": "rag",
"task_id": "rag-218",
"ok": true,
"answer": "1",
"wall_s": 0.038,
"tokens": 2,
"error": null
},
{
"role": "rag",
"task_id": "rag-219",
"ok": true,
"answer": "card 0, 16k",
"wall_s": 0.133,
"tokens": 8,
"error": null
},
{
"role": "rag",
"task_id": "rag-220",
"ok": true,
"answer": "16k",
"wall_s": 0.054,
"tokens": 3,
"error": null
},
{
"role": "rag",
"task_id": "rag-221",
"ok": true,
"answer": "card 1, 8k",
"wall_s": 0.132,
"tokens": 8,
"error": null
},
{
"role": "rag",
"task_id": "rag-222",
"ok": true,
"answer": "2",
"wall_s": 0.038,
"tokens": 2,
"error": null
},
{
"role": "rag",
"task_id": "rag-223",
"ok": true,
"answer": "8k",
"wall_s": 0.054,
"tokens": 3,
"error": null
},
{
"role": "rag",
"task_id": "rag-224",
"ok": true,
"answer": "8k",
"wall_s": 0.054,
"tokens": 3,
"error": null
},
{
"role": "rag",
"task_id": "rag-225",
"ok": true,
"answer": "9.54",
"wall_s": 0.07,
"tokens": 4,
"error": null
},
{
"role": "rag",
"task_id": "rag-226",
"ok": true,
"answer": "32k",
"wall_s": 0.054,
"tokens": 3,
"error": null
},
{
"role": "rag",
"task_id": "rag-227",
"ok": true,
"answer": "card 1, 64k",
"wall_s": 0.133,
"tokens": 8,
"error": null
},
{
"role": "rag",
"task_id": "rag-228",
"ok": true,
"answer": "card 1, 8k",
"wall_s": 0.133,
"tokens": 8,
"error": null
},
{
"role": "rag",
"task_id": "rag-229",
"ok": true,
"answer": "32k",
"wall_s": 0.054,
"tokens": 3,
"error": null
},
{
"role": "rag",
"task_id": "rag-230",
"ok": true,
"answer": "0",
"wall_s": 0.038,
"tokens": 2,
"error": null
},
{
"role": "rag",
"task_id": "rag-231",
"ok": true,
"answer": "8k",
"wall_s": 0.054,
"tokens": 3,
"error": null
},
{
"role": "rag",
"task_id": "rag-232",
"ok": true,
"answer": "5.69",
"wall_s": 0.07,
"tokens": 4,
"error": null
},
{
"role": "rag",
"task_id": "rag-233",
"ok": true,
"answer": "4.99",
"wall_s": 0.07,
"tokens": 4,
"error": null
},
{
"role": "rag",
"task_id": "rag-234",
"ok": true,
"answer": "card 1, 8k",
"wall_s": 0.132,
"tokens": 8,
"error": null
},
{
"role": "rag",
"task_id": "rag-235",
"ok": true,
"answer": "2",
"wall_s": 0.038,
"tokens": 2,
"error": null
},
{
"role": "rag",
"task_id": "rag-236",
"ok": true,
"answer": "32k",
"wall_s": 0.054,
"tokens": 3,
"error": null
},
{
"role": "rag",
"task_id": "rag-237",
"ok": true,
"answer": "8k",
"wall_s": 0.054,
"tokens": 3,
"error": null
},
{
"role": "rag",
"task_id": "rag-238",
"ok": true,
"answer": "card 0, 32k",
"wall_s": 0.133,
"tokens": 8,
"error": null
},
{
"role": "rag",
"task_id": "rag-239",
"ok": true,
"answer": "2",
"wall_s": 0.038,
"tokens": 2,
"error": null
}
]
}