gua / temperatures.json
chenyvathf's picture
gua v2: continue v1 on 12,000 balanced questions from macos-cua v3 (adds Logic Pro, OBS, Plasticity)
1357427 verified
Raw History Blame Contribute Delete
1.41 kB
{
"dev": "experiments/decision/dev-v3-lora-v3.jsonl",
"eval": "experiments/decision/eval-v3-lora-v3.jsonl",
"questions": {
"grounding": {
"temperature": 1.25,
"dev_items": 2555,
"dev_ece": {
"before": 0.2877,
"after": 0.2073
},
"dev_log_loss": {
"before": 1.0868,
"after": 1.0408
},
"eval_items": 6508,
"eval_ece": {
"before": 0.259,
"after": 0.1725
},
"eval_log_loss": {
"before": 1.1036,
"after": 1.0632
}
},
"action": {
"temperature": 2.15,
"dev_items": 4006,
"dev_ece": {
"before": 0.3158,
"after": 0.0655
},
"dev_log_loss": {
"before": 1.9669,
"after": 1.4436
},
"eval_items": 9972,
"eval_ece": {
"before": 0.2941,
"after": 0.0603
},
"eval_log_loss": {
"before": 1.9069,
"after": 1.4122
}
},
"next_target": {
"temperature": 1.2,
"dev_items": 2555,
"dev_ece": {
"before": 0.2854,
"after": 0.1744
},
"dev_log_loss": {
"before": 2.5119,
"after": 2.4338
},
"eval_items": 6508,
"eval_ece": {
"before": 0.2527,
"after": 0.1405
},
"eval_log_loss": {
"before": 2.5645,
"after": 2.483
}
}
}
}