nima / training_metadata.json
neimand's picture
Upload folder using huggingface_hub
104376c verified
Raw History Blame Contribute Delete
8.02 kB
{
"best_epoch": 1,
"selection_nll": 0.43172744103717203,
"versions": {
"mlx": "0.32.3",
"mlx-lm": "0.32.0",
"numpy": "2.5.3"
},
"effective_training_rows": 1706,
"optimizer_updates": 17120,
"seconds": 89.4501815829999,
"memory": {
"mlx_peak_allocated_bytes": 87674924,
"mlx_active_allocated_bytes": 21107772,
"process_max_rss_bytes_macos": 1636679680
},
"runtime_checks": {
"variable_candidates": [
2,
5,
8
],
"padding_isolation": true,
"candidate_permutation_equivariance": true,
"bidirectional_candidate_interaction": true,
"evidence_influence": true,
"finite_gradients": true,
"gradient_norms": {
"evidence_attention": 1.0483995552985155,
"joint": 3.5755980015137983,
"project": 19.5132737136767,
"score": 7.060013324843847
},
"reload_agreement": true,
"trainable_parameters": 1318657
},
"validation": {
"selection": {
"count": 256,
"unique_requests": 128,
"accuracy": 0.8359375,
"correct": 214,
"nll": 0.43172744103717203,
"by_tool": {
"web_search": {
"count": 32,
"correct": 32
},
"rag_search": {
"count": 32,
"correct": 24
},
"database_search": {
"count": 32,
"correct": 32
},
"send_email": {
"count": 32,
"correct": 19
},
"weather_search": {
"count": 32,
"correct": 31
},
"calendar_search": {
"count": 32,
"correct": 32
},
"calculator": {
"count": 32,
"correct": 32
},
"no_tool": {
"count": 32,
"correct": 12
}
},
"confusion": [
{
"target": "calculator",
"prediction": "calculator",
"count": 32
},
{
"target": "calendar_search",
"prediction": "calendar_search",
"count": 32
},
{
"target": "database_search",
"prediction": "database_search",
"count": 32
},
{
"target": "no_tool",
"prediction": "database_search",
"count": 6
},
{
"target": "no_tool",
"prediction": "no_tool",
"count": 12
},
{
"target": "no_tool",
"prediction": "send_email",
"count": 10
},
{
"target": "no_tool",
"prediction": "weather_search",
"count": 4
},
{
"target": "rag_search",
"prediction": "database_search",
"count": 4
},
{
"target": "rag_search",
"prediction": "no_tool",
"count": 4
},
{
"target": "rag_search",
"prediction": "rag_search",
"count": 24
},
{
"target": "send_email",
"prediction": "database_search",
"count": 1
},
{
"target": "send_email",
"prediction": "no_tool",
"count": 12
},
{
"target": "send_email",
"prediction": "send_email",
"count": 19
},
{
"target": "weather_search",
"prediction": "no_tool",
"count": 1
},
{
"target": "weather_search",
"prediction": "weather_search",
"count": 31
},
{
"target": "web_search",
"prediction": "web_search",
"count": 32
}
],
"no_tool": {
"precision": 0.41379310344827586,
"recall": 0.375,
"false_abstentions": 17
},
"orders_disagree_requests": 21,
"all_views_correct_requests": 98
},
"calibration": {
"count": 128,
"unique_requests": 128,
"accuracy": 0.796875,
"correct": 102,
"nll": 0.5966955877251904,
"by_tool": {
"web_search": {
"count": 16,
"correct": 15
},
"rag_search": {
"count": 16,
"correct": 5
},
"database_search": {
"count": 16,
"correct": 14
},
"send_email": {
"count": 16,
"correct": 9
},
"weather_search": {
"count": 16,
"correct": 15
},
"calendar_search": {
"count": 16,
"correct": 16
},
"calculator": {
"count": 16,
"correct": 16
},
"no_tool": {
"count": 16,
"correct": 12
}
},
"confusion": [
{
"target": "calculator",
"prediction": "calculator",
"count": 16
},
{
"target": "calendar_search",
"prediction": "calendar_search",
"count": 16
},
{
"target": "database_search",
"prediction": "database_search",
"count": 14
},
{
"target": "database_search",
"prediction": "no_tool",
"count": 2
},
{
"target": "no_tool",
"prediction": "database_search",
"count": 1
},
{
"target": "no_tool",
"prediction": "no_tool",
"count": 12
},
{
"target": "no_tool",
"prediction": "send_email",
"count": 3
},
{
"target": "rag_search",
"prediction": "database_search",
"count": 7
},
{
"target": "rag_search",
"prediction": "no_tool",
"count": 4
},
{
"target": "rag_search",
"prediction": "rag_search",
"count": 5
},
{
"target": "send_email",
"prediction": "no_tool",
"count": 7
},
{
"target": "send_email",
"prediction": "send_email",
"count": 9
},
{
"target": "weather_search",
"prediction": "no_tool",
"count": 1
},
{
"target": "weather_search",
"prediction": "weather_search",
"count": 15
},
{
"target": "web_search",
"prediction": "no_tool",
"count": 1
},
{
"target": "web_search",
"prediction": "web_search",
"count": 15
}
],
"no_tool": {
"precision": 0.4444444444444444,
"recall": 0.75,
"false_abstentions": 15
},
"orders_disagree_requests": 0,
"all_views_correct_requests": 102
}
},
"head_sha256": "0e8f91e97b1ac2e309463fbb8b0e10c9d165bf4961bd40613aaa0603830e2988",
"calibration_policy": "temperature fixed at 1; no thresholds or calibration fitting in this milestone",
"source_hashes": {
"run.py": "6e8d151b03c387153bbcf229f91ffa3fe1e05f042f1e271ba0aaa22155801f5b",
"validate_dataset.py": "a811abc97d54b8a1ca5e48a2f12c5591d3300685f7faf7e4c2b61161172a7c22",
"test_offline.py": "1cd7a07e215240161a989e35ccddf45029bdc044d67250a0fdf9cc59a32249fa",
"architecture.json": "46918f976e86089108003daf0c7ece7ef4cdaf677e3ae5b8a0dbb8e19989650b",
"head.py": "5f2ddf1b5b45b0483bd62fc5c86e9cba1f283e854548abdad9bcda6a5e47556c",
"protocol.json": "b7376ce11843210d72a2ffe129bf193af2bdf05fdb9bcf8cc74d8f2b6fe161c6",
"CPU_CHECKS.json": "fe15fc54746af9d40ac37fea8c9ecdaa1e32d77ff7aae7fce0819f43acb0cac2",
"workflow.py": "88b481c52a96a725c800a3f6eb40f70ced04c5ccaadefb1f1187e0d296c4eb36",
"build_data.py": "4a19cc1f4175b43dad77eebdc17078565f05b86b21cdd082277ca507f1f6d717",
"evaluate_pair.py": "5d2a55580220ff43bef4224824c5895a8ebd306f707a6e47b7a5463a9b7cf1d3",
"data.py": "bd1bdb996f2d2de4a2c993bd2cdcb37d0c1d975b604f400c8bd21f3d3df93695"
}
}