| { |
| "examples": 160, |
| "seconds": 2.227508544921875, |
| "decoding": "raw greedy, no repair, no teacher fallback", |
| "checkpoint_sha256": "9489b5cde69c11f64b3e7031182e8248aa83790f1ad4aeefbf99a4e5e152e692", |
| "sql_metric": "Compile against the supplied schema, then compare results on two generated SQLite fixtures after dialect adaptation; not native MySQL/PostgreSQL execution.", |
| "metrics": { |
| "json_valid": { |
| "correct": 160, |
| "total": 160, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 159, |
| "total": 160, |
| "rate": 0.99375 |
| }, |
| "action_correct": { |
| "correct": 159, |
| "total": 160, |
| "rate": 0.99375 |
| }, |
| "tool_correct": { |
| "correct": 146, |
| "total": 152, |
| "rate": 0.9605263157894737 |
| }, |
| "arguments_exact": { |
| "correct": 124, |
| "total": 152, |
| "rate": 0.8157894736842105 |
| }, |
| "sql_equivalent": { |
| "correct": 94, |
| "total": 112, |
| "rate": 0.8392857142857143 |
| }, |
| "success": { |
| "correct": 132, |
| "total": 160, |
| "rate": 0.825 |
| } |
| }, |
| "groups": { |
| "language:noisy_en": { |
| "json_valid": { |
| "correct": 40, |
| "total": 40, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 40, |
| "total": 40, |
| "rate": 1.0 |
| }, |
| "action_correct": { |
| "correct": 39, |
| "total": 40, |
| "rate": 0.975 |
| }, |
| "tool_correct": { |
| "correct": 35, |
| "total": 38, |
| "rate": 0.9210526315789473 |
| }, |
| "arguments_exact": { |
| "correct": 31, |
| "total": 38, |
| "rate": 0.8157894736842105 |
| }, |
| "sql_equivalent": { |
| "correct": 23, |
| "total": 28, |
| "rate": 0.8214285714285714 |
| }, |
| "success": { |
| "correct": 33, |
| "total": 40, |
| "rate": 0.825 |
| } |
| }, |
| "backend:supabase": { |
| "json_valid": { |
| "correct": 80, |
| "total": 80, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 80, |
| "total": 80, |
| "rate": 1.0 |
| }, |
| "action_correct": { |
| "correct": 79, |
| "total": 80, |
| "rate": 0.9875 |
| }, |
| "tool_correct": { |
| "correct": 73, |
| "total": 76, |
| "rate": 0.9605263157894737 |
| }, |
| "arguments_exact": { |
| "correct": 59, |
| "total": 76, |
| "rate": 0.7763157894736842 |
| }, |
| "sql_equivalent": { |
| "correct": 46, |
| "total": 56, |
| "rate": 0.8214285714285714 |
| }, |
| "success": { |
| "correct": 63, |
| "total": 80, |
| "rate": 0.7875 |
| } |
| }, |
| "operation:all": { |
| "json_valid": { |
| "correct": 8, |
| "total": 8, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 8, |
| "total": 8, |
| "rate": 1.0 |
| }, |
| "action_correct": { |
| "correct": 7, |
| "total": 8, |
| "rate": 0.875 |
| }, |
| "tool_correct": { |
| "correct": 7, |
| "total": 8, |
| "rate": 0.875 |
| }, |
| "arguments_exact": { |
| "correct": 5, |
| "total": 8, |
| "rate": 0.625 |
| }, |
| "sql_equivalent": { |
| "correct": 5, |
| "total": 8, |
| "rate": 0.625 |
| }, |
| "success": { |
| "correct": 5, |
| "total": 8, |
| "rate": 0.625 |
| } |
| }, |
| "operation:null": { |
| "json_valid": { |
| "correct": 8, |
| "total": 8, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 8, |
| "total": 8, |
| "rate": 1.0 |
| }, |
| "action_correct": { |
| "correct": 8, |
| "total": 8, |
| "rate": 1.0 |
| }, |
| "tool_correct": { |
| "correct": 8, |
| "total": 8, |
| "rate": 1.0 |
| }, |
| "arguments_exact": { |
| "correct": 2, |
| "total": 8, |
| "rate": 0.25 |
| }, |
| "sql_equivalent": { |
| "correct": 2, |
| "total": 8, |
| "rate": 0.25 |
| }, |
| "success": { |
| "correct": 2, |
| "total": 8, |
| "rate": 0.25 |
| } |
| }, |
| "operation:gte": { |
| "json_valid": { |
| "correct": 8, |
| "total": 8, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 8, |
| "total": 8, |
| "rate": 1.0 |
| }, |
| "action_correct": { |
| "correct": 8, |
| "total": 8, |
| "rate": 1.0 |
| }, |
| "tool_correct": { |
| "correct": 8, |
| "total": 8, |
| "rate": 1.0 |
| }, |
| "arguments_exact": { |
| "correct": 7, |
| "total": 8, |
| "rate": 0.875 |
| }, |
| "sql_equivalent": { |
| "correct": 7, |
| "total": 8, |
| "rate": 0.875 |
| }, |
| "success": { |
| "correct": 7, |
| "total": 8, |
| "rate": 0.875 |
| } |
| }, |
| "language:en": { |
| "json_valid": { |
| "correct": 40, |
| "total": 40, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 39, |
| "total": 40, |
| "rate": 0.975 |
| }, |
| "action_correct": { |
| "correct": 40, |
| "total": 40, |
| "rate": 1.0 |
| }, |
| "tool_correct": { |
| "correct": 37, |
| "total": 38, |
| "rate": 0.9736842105263158 |
| }, |
| "arguments_exact": { |
| "correct": 25, |
| "total": 38, |
| "rate": 0.6578947368421053 |
| }, |
| "sql_equivalent": { |
| "correct": 20, |
| "total": 28, |
| "rate": 0.7142857142857143 |
| }, |
| "success": { |
| "correct": 27, |
| "total": 40, |
| "rate": 0.675 |
| } |
| }, |
| "operation:project": { |
| "json_valid": { |
| "correct": 8, |
| "total": 8, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 8, |
| "total": 8, |
| "rate": 1.0 |
| }, |
| "action_correct": { |
| "correct": 8, |
| "total": 8, |
| "rate": 1.0 |
| }, |
| "tool_correct": { |
| "correct": 8, |
| "total": 8, |
| "rate": 1.0 |
| }, |
| "arguments_exact": { |
| "correct": 8, |
| "total": 8, |
| "rate": 1.0 |
| }, |
| "sql_equivalent": { |
| "correct": 8, |
| "total": 8, |
| "rate": 1.0 |
| }, |
| "success": { |
| "correct": 8, |
| "total": 8, |
| "rate": 1.0 |
| } |
| }, |
| "language:hinglish": { |
| "json_valid": { |
| "correct": 40, |
| "total": 40, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 40, |
| "total": 40, |
| "rate": 1.0 |
| }, |
| "action_correct": { |
| "correct": 40, |
| "total": 40, |
| "rate": 1.0 |
| }, |
| "tool_correct": { |
| "correct": 38, |
| "total": 38, |
| "rate": 1.0 |
| }, |
| "arguments_exact": { |
| "correct": 34, |
| "total": 38, |
| "rate": 0.8947368421052632 |
| }, |
| "sql_equivalent": { |
| "correct": 25, |
| "total": 28, |
| "rate": 0.8928571428571429 |
| }, |
| "success": { |
| "correct": 36, |
| "total": 40, |
| "rate": 0.9 |
| } |
| }, |
| "operation:sum": { |
| "json_valid": { |
| "correct": 8, |
| "total": 8, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 8, |
| "total": 8, |
| "rate": 1.0 |
| }, |
| "action_correct": { |
| "correct": 8, |
| "total": 8, |
| "rate": 1.0 |
| }, |
| "tool_correct": { |
| "correct": 8, |
| "total": 8, |
| "rate": 1.0 |
| }, |
| "arguments_exact": { |
| "correct": 6, |
| "total": 8, |
| "rate": 0.75 |
| }, |
| "sql_equivalent": { |
| "correct": 6, |
| "total": 8, |
| "rate": 0.75 |
| }, |
| "success": { |
| "correct": 6, |
| "total": 8, |
| "rate": 0.75 |
| } |
| }, |
| "operation:distinct": { |
| "json_valid": { |
| "correct": 8, |
| "total": 8, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 8, |
| "total": 8, |
| "rate": 1.0 |
| }, |
| "action_correct": { |
| "correct": 8, |
| "total": 8, |
| "rate": 1.0 |
| }, |
| "tool_correct": { |
| "correct": 8, |
| "total": 8, |
| "rate": 1.0 |
| }, |
| "arguments_exact": { |
| "correct": 8, |
| "total": 8, |
| "rate": 1.0 |
| }, |
| "sql_equivalent": { |
| "correct": 8, |
| "total": 8, |
| "rate": 1.0 |
| }, |
| "success": { |
| "correct": 8, |
| "total": 8, |
| "rate": 1.0 |
| } |
| }, |
| "operation:project_eq": { |
| "json_valid": { |
| "correct": 8, |
| "total": 8, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 8, |
| "total": 8, |
| "rate": 1.0 |
| }, |
| "action_correct": { |
| "correct": 8, |
| "total": 8, |
| "rate": 1.0 |
| }, |
| "tool_correct": { |
| "correct": 8, |
| "total": 8, |
| "rate": 1.0 |
| }, |
| "arguments_exact": { |
| "correct": 8, |
| "total": 8, |
| "rate": 1.0 |
| }, |
| "sql_equivalent": { |
| "correct": 8, |
| "total": 8, |
| "rate": 1.0 |
| }, |
| "success": { |
| "correct": 8, |
| "total": 8, |
| "rate": 1.0 |
| } |
| }, |
| "operation:lte": { |
| "json_valid": { |
| "correct": 8, |
| "total": 8, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 8, |
| "total": 8, |
| "rate": 1.0 |
| }, |
| "action_correct": { |
| "correct": 8, |
| "total": 8, |
| "rate": 1.0 |
| }, |
| "tool_correct": { |
| "correct": 8, |
| "total": 8, |
| "rate": 1.0 |
| }, |
| "arguments_exact": { |
| "correct": 4, |
| "total": 8, |
| "rate": 0.5 |
| }, |
| "sql_equivalent": { |
| "correct": 4, |
| "total": 8, |
| "rate": 0.5 |
| }, |
| "success": { |
| "correct": 4, |
| "total": 8, |
| "rate": 0.5 |
| } |
| }, |
| "language:hi": { |
| "json_valid": { |
| "correct": 40, |
| "total": 40, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 40, |
| "total": 40, |
| "rate": 1.0 |
| }, |
| "action_correct": { |
| "correct": 40, |
| "total": 40, |
| "rate": 1.0 |
| }, |
| "tool_correct": { |
| "correct": 36, |
| "total": 38, |
| "rate": 0.9473684210526315 |
| }, |
| "arguments_exact": { |
| "correct": 34, |
| "total": 38, |
| "rate": 0.8947368421052632 |
| }, |
| "sql_equivalent": { |
| "correct": 26, |
| "total": 28, |
| "rate": 0.9285714285714286 |
| }, |
| "success": { |
| "correct": 36, |
| "total": 40, |
| "rate": 0.9 |
| } |
| }, |
| "operation:list_tables": { |
| "json_valid": { |
| "correct": 8, |
| "total": 8, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 7, |
| "total": 8, |
| "rate": 0.875 |
| }, |
| "action_correct": { |
| "correct": 8, |
| "total": 8, |
| "rate": 1.0 |
| }, |
| "tool_correct": { |
| "correct": 3, |
| "total": 8, |
| "rate": 0.375 |
| }, |
| "arguments_exact": { |
| "correct": 1, |
| "total": 8, |
| "rate": 0.125 |
| }, |
| "sql_equivalent": { |
| "correct": 0, |
| "total": 0, |
| "rate": 0.0 |
| }, |
| "success": { |
| "correct": 1, |
| "total": 8, |
| "rate": 0.125 |
| } |
| }, |
| "operation:ambiguous": { |
| "json_valid": { |
| "correct": 8, |
| "total": 8, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 8, |
| "total": 8, |
| "rate": 1.0 |
| }, |
| "action_correct": { |
| "correct": 8, |
| "total": 8, |
| "rate": 1.0 |
| }, |
| "tool_correct": { |
| "correct": 0, |
| "total": 0, |
| "rate": 0.0 |
| }, |
| "arguments_exact": { |
| "correct": 0, |
| "total": 0, |
| "rate": 0.0 |
| }, |
| "sql_equivalent": { |
| "correct": 0, |
| "total": 0, |
| "rate": 0.0 |
| }, |
| "success": { |
| "correct": 8, |
| "total": 8, |
| "rate": 1.0 |
| } |
| }, |
| "operation:lt": { |
| "json_valid": { |
| "correct": 8, |
| "total": 8, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 8, |
| "total": 8, |
| "rate": 1.0 |
| }, |
| "action_correct": { |
| "correct": 8, |
| "total": 8, |
| "rate": 1.0 |
| }, |
| "tool_correct": { |
| "correct": 8, |
| "total": 8, |
| "rate": 1.0 |
| }, |
| "arguments_exact": { |
| "correct": 7, |
| "total": 8, |
| "rate": 0.875 |
| }, |
| "sql_equivalent": { |
| "correct": 8, |
| "total": 8, |
| "rate": 1.0 |
| }, |
| "success": { |
| "correct": 7, |
| "total": 8, |
| "rate": 0.875 |
| } |
| }, |
| "operation:gt": { |
| "json_valid": { |
| "correct": 8, |
| "total": 8, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 8, |
| "total": 8, |
| "rate": 1.0 |
| }, |
| "action_correct": { |
| "correct": 8, |
| "total": 8, |
| "rate": 1.0 |
| }, |
| "tool_correct": { |
| "correct": 8, |
| "total": 8, |
| "rate": 1.0 |
| }, |
| "arguments_exact": { |
| "correct": 7, |
| "total": 8, |
| "rate": 0.875 |
| }, |
| "sql_equivalent": { |
| "correct": 8, |
| "total": 8, |
| "rate": 1.0 |
| }, |
| "success": { |
| "correct": 7, |
| "total": 8, |
| "rate": 0.875 |
| } |
| }, |
| "operation:avg": { |
| "json_valid": { |
| "correct": 8, |
| "total": 8, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 8, |
| "total": 8, |
| "rate": 1.0 |
| }, |
| "action_correct": { |
| "correct": 8, |
| "total": 8, |
| "rate": 1.0 |
| }, |
| "tool_correct": { |
| "correct": 8, |
| "total": 8, |
| "rate": 1.0 |
| }, |
| "arguments_exact": { |
| "correct": 7, |
| "total": 8, |
| "rate": 0.875 |
| }, |
| "sql_equivalent": { |
| "correct": 8, |
| "total": 8, |
| "rate": 1.0 |
| }, |
| "success": { |
| "correct": 7, |
| "total": 8, |
| "rate": 0.875 |
| } |
| }, |
| "operation:count": { |
| "json_valid": { |
| "correct": 8, |
| "total": 8, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 8, |
| "total": 8, |
| "rate": 1.0 |
| }, |
| "action_correct": { |
| "correct": 8, |
| "total": 8, |
| "rate": 1.0 |
| }, |
| "tool_correct": { |
| "correct": 8, |
| "total": 8, |
| "rate": 1.0 |
| }, |
| "arguments_exact": { |
| "correct": 8, |
| "total": 8, |
| "rate": 1.0 |
| }, |
| "sql_equivalent": { |
| "correct": 8, |
| "total": 8, |
| "rate": 1.0 |
| }, |
| "success": { |
| "correct": 8, |
| "total": 8, |
| "rate": 1.0 |
| } |
| }, |
| "operation:group_count": { |
| "json_valid": { |
| "correct": 8, |
| "total": 8, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 8, |
| "total": 8, |
| "rate": 1.0 |
| }, |
| "action_correct": { |
| "correct": 8, |
| "total": 8, |
| "rate": 1.0 |
| }, |
| "tool_correct": { |
| "correct": 8, |
| "total": 8, |
| "rate": 1.0 |
| }, |
| "arguments_exact": { |
| "correct": 6, |
| "total": 8, |
| "rate": 0.75 |
| }, |
| "sql_equivalent": { |
| "correct": 6, |
| "total": 8, |
| "rate": 0.75 |
| }, |
| "success": { |
| "correct": 6, |
| "total": 8, |
| "rate": 0.75 |
| } |
| }, |
| "operation:top": { |
| "json_valid": { |
| "correct": 8, |
| "total": 8, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 8, |
| "total": 8, |
| "rate": 1.0 |
| }, |
| "action_correct": { |
| "correct": 8, |
| "total": 8, |
| "rate": 1.0 |
| }, |
| "tool_correct": { |
| "correct": 8, |
| "total": 8, |
| "rate": 1.0 |
| }, |
| "arguments_exact": { |
| "correct": 8, |
| "total": 8, |
| "rate": 1.0 |
| }, |
| "sql_equivalent": { |
| "correct": 8, |
| "total": 8, |
| "rate": 1.0 |
| }, |
| "success": { |
| "correct": 8, |
| "total": 8, |
| "rate": 1.0 |
| } |
| }, |
| "backend:mysql": { |
| "json_valid": { |
| "correct": 80, |
| "total": 80, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 79, |
| "total": 80, |
| "rate": 0.9875 |
| }, |
| "action_correct": { |
| "correct": 80, |
| "total": 80, |
| "rate": 1.0 |
| }, |
| "tool_correct": { |
| "correct": 73, |
| "total": 76, |
| "rate": 0.9605263157894737 |
| }, |
| "arguments_exact": { |
| "correct": 65, |
| "total": 76, |
| "rate": 0.8552631578947368 |
| }, |
| "sql_equivalent": { |
| "correct": 48, |
| "total": 56, |
| "rate": 0.8571428571428571 |
| }, |
| "success": { |
| "correct": 69, |
| "total": 80, |
| "rate": 0.8625 |
| } |
| }, |
| "operation:weather": { |
| "json_valid": { |
| "correct": 8, |
| "total": 8, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 8, |
| "total": 8, |
| "rate": 1.0 |
| }, |
| "action_correct": { |
| "correct": 8, |
| "total": 8, |
| "rate": 1.0 |
| }, |
| "tool_correct": { |
| "correct": 8, |
| "total": 8, |
| "rate": 1.0 |
| }, |
| "arguments_exact": { |
| "correct": 8, |
| "total": 8, |
| "rate": 1.0 |
| }, |
| "sql_equivalent": { |
| "correct": 0, |
| "total": 0, |
| "rate": 0.0 |
| }, |
| "success": { |
| "correct": 8, |
| "total": 8, |
| "rate": 1.0 |
| } |
| }, |
| "operation:search": { |
| "json_valid": { |
| "correct": 8, |
| "total": 8, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 8, |
| "total": 8, |
| "rate": 1.0 |
| }, |
| "action_correct": { |
| "correct": 8, |
| "total": 8, |
| "rate": 1.0 |
| }, |
| "tool_correct": { |
| "correct": 8, |
| "total": 8, |
| "rate": 1.0 |
| }, |
| "arguments_exact": { |
| "correct": 8, |
| "total": 8, |
| "rate": 1.0 |
| }, |
| "sql_equivalent": { |
| "correct": 0, |
| "total": 0, |
| "rate": 0.0 |
| }, |
| "success": { |
| "correct": 8, |
| "total": 8, |
| "rate": 1.0 |
| } |
| }, |
| "operation:ticket": { |
| "json_valid": { |
| "correct": 8, |
| "total": 8, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 8, |
| "total": 8, |
| "rate": 1.0 |
| }, |
| "action_correct": { |
| "correct": 8, |
| "total": 8, |
| "rate": 1.0 |
| }, |
| "tool_correct": { |
| "correct": 8, |
| "total": 8, |
| "rate": 1.0 |
| }, |
| "arguments_exact": { |
| "correct": 8, |
| "total": 8, |
| "rate": 1.0 |
| }, |
| "sql_equivalent": { |
| "correct": 0, |
| "total": 0, |
| "rate": 0.0 |
| }, |
| "success": { |
| "correct": 8, |
| "total": 8, |
| "rate": 1.0 |
| } |
| }, |
| "operation:read_file": { |
| "json_valid": { |
| "correct": 8, |
| "total": 8, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 8, |
| "total": 8, |
| "rate": 1.0 |
| }, |
| "action_correct": { |
| "correct": 8, |
| "total": 8, |
| "rate": 1.0 |
| }, |
| "tool_correct": { |
| "correct": 8, |
| "total": 8, |
| "rate": 1.0 |
| }, |
| "arguments_exact": { |
| "correct": 8, |
| "total": 8, |
| "rate": 1.0 |
| }, |
| "sql_equivalent": { |
| "correct": 0, |
| "total": 0, |
| "rate": 0.0 |
| }, |
| "success": { |
| "correct": 8, |
| "total": 8, |
| "rate": 1.0 |
| } |
| } |
| } |
| } |
