{ "best_epoch": 1, "selection_nll": 0.43172744103717203, "versions": { "mlx": "0.32.3", "mlx-lm": "0.32.0", "numpy": "2.5.3" }, "effective_training_rows": 1706, "optimizer_updates": 17120, "seconds": 89.4501815829999, "memory": { "mlx_peak_allocated_bytes": 87674924, "mlx_active_allocated_bytes": 21107772, "process_max_rss_bytes_macos": 1636679680 }, "runtime_checks": { "variable_candidates": [ 2, 5, 8 ], "padding_isolation": true, "candidate_permutation_equivariance": true, "bidirectional_candidate_interaction": true, "evidence_influence": true, "finite_gradients": true, "gradient_norms": { "evidence_attention": 1.0483995552985155, "joint": 3.5755980015137983, "project": 19.5132737136767, "score": 7.060013324843847 }, "reload_agreement": true, "trainable_parameters": 1318657 }, "validation": { "selection": { "count": 256, "unique_requests": 128, "accuracy": 0.8359375, "correct": 214, "nll": 0.43172744103717203, "by_tool": { "web_search": { "count": 32, "correct": 32 }, "rag_search": { "count": 32, "correct": 24 }, "database_search": { "count": 32, "correct": 32 }, "send_email": { "count": 32, "correct": 19 }, "weather_search": { "count": 32, "correct": 31 }, "calendar_search": { "count": 32, "correct": 32 }, "calculator": { "count": 32, "correct": 32 }, "no_tool": { "count": 32, "correct": 12 } }, "confusion": [ { "target": "calculator", "prediction": "calculator", "count": 32 }, { "target": "calendar_search", "prediction": "calendar_search", "count": 32 }, { "target": "database_search", "prediction": "database_search", "count": 32 }, { "target": "no_tool", "prediction": "database_search", "count": 6 }, { "target": "no_tool", "prediction": "no_tool", "count": 12 }, { "target": "no_tool", "prediction": "send_email", "count": 10 }, { "target": "no_tool", "prediction": "weather_search", "count": 4 }, { "target": "rag_search", "prediction": "database_search", "count": 4 }, { "target": "rag_search", "prediction": "no_tool", "count": 4 }, { "target": "rag_search", "prediction": "rag_search", "count": 24 }, { "target": "send_email", "prediction": "database_search", "count": 1 }, { "target": "send_email", "prediction": "no_tool", "count": 12 }, { "target": "send_email", "prediction": "send_email", "count": 19 }, { "target": "weather_search", "prediction": "no_tool", "count": 1 }, { "target": "weather_search", "prediction": "weather_search", "count": 31 }, { "target": "web_search", "prediction": "web_search", "count": 32 } ], "no_tool": { "precision": 0.41379310344827586, "recall": 0.375, "false_abstentions": 17 }, "orders_disagree_requests": 21, "all_views_correct_requests": 98 }, "calibration": { "count": 128, "unique_requests": 128, "accuracy": 0.796875, "correct": 102, "nll": 0.5966955877251904, "by_tool": { "web_search": { "count": 16, "correct": 15 }, "rag_search": { "count": 16, "correct": 5 }, "database_search": { "count": 16, "correct": 14 }, "send_email": { "count": 16, "correct": 9 }, "weather_search": { "count": 16, "correct": 15 }, "calendar_search": { "count": 16, "correct": 16 }, "calculator": { "count": 16, "correct": 16 }, "no_tool": { "count": 16, "correct": 12 } }, "confusion": [ { "target": "calculator", "prediction": "calculator", "count": 16 }, { "target": "calendar_search", "prediction": "calendar_search", "count": 16 }, { "target": "database_search", "prediction": "database_search", "count": 14 }, { "target": "database_search", "prediction": "no_tool", "count": 2 }, { "target": "no_tool", "prediction": "database_search", "count": 1 }, { "target": "no_tool", "prediction": "no_tool", "count": 12 }, { "target": "no_tool", "prediction": "send_email", "count": 3 }, { "target": "rag_search", "prediction": "database_search", "count": 7 }, { "target": "rag_search", "prediction": "no_tool", "count": 4 }, { "target": "rag_search", "prediction": "rag_search", "count": 5 }, { "target": "send_email", "prediction": "no_tool", "count": 7 }, { "target": "send_email", "prediction": "send_email", "count": 9 }, { "target": "weather_search", "prediction": "no_tool", "count": 1 }, { "target": "weather_search", "prediction": "weather_search", "count": 15 }, { "target": "web_search", "prediction": "no_tool", "count": 1 }, { "target": "web_search", "prediction": "web_search", "count": 15 } ], "no_tool": { "precision": 0.4444444444444444, "recall": 0.75, "false_abstentions": 15 }, "orders_disagree_requests": 0, "all_views_correct_requests": 102 } }, "head_sha256": "0e8f91e97b1ac2e309463fbb8b0e10c9d165bf4961bd40613aaa0603830e2988", "calibration_policy": "temperature fixed at 1; no thresholds or calibration fitting in this milestone", "source_hashes": { "run.py": "6e8d151b03c387153bbcf229f91ffa3fe1e05f042f1e271ba0aaa22155801f5b", "validate_dataset.py": "a811abc97d54b8a1ca5e48a2f12c5591d3300685f7faf7e4c2b61161172a7c22", "test_offline.py": "1cd7a07e215240161a989e35ccddf45029bdc044d67250a0fdf9cc59a32249fa", "architecture.json": "46918f976e86089108003daf0c7ece7ef4cdaf677e3ae5b8a0dbb8e19989650b", "head.py": "5f2ddf1b5b45b0483bd62fc5c86e9cba1f283e854548abdad9bcda6a5e47556c", "protocol.json": "b7376ce11843210d72a2ffe129bf193af2bdf05fdb9bcf8cc74d8f2b6fe161c6", "CPU_CHECKS.json": "fe15fc54746af9d40ac37fea8c9ecdaa1e32d77ff7aae7fce0819f43acb0cac2", "workflow.py": "88b481c52a96a725c800a3f6eb40f70ced04c5ccaadefb1f1187e0d296c4eb36", "build_data.py": "4a19cc1f4175b43dad77eebdc17078565f05b86b21cdd082277ca507f1f6d717", "evaluate_pair.py": "5d2a55580220ff43bef4224824c5895a8ebd306f707a6e47b7a5463a9b7cf1d3", "data.py": "bd1bdb996f2d2de4a2c993bd2cdcb37d0c1d975b604f400c8bd21f3d3df93695" } }