| { |
| "baseline": "TF-IDF nearest question plus tool-description and DDL-position binding; no expected answer or gold slots used", |
| "training_questions": 212073, |
| "examples": 1196, |
| "seconds": 9.818428039550781, |
| "metrics": { |
| "json_valid": { |
| "correct": 1196, |
| "total": 1196, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 1187, |
| "total": 1196, |
| "rate": 0.9924749163879598 |
| }, |
| "action_correct": { |
| "correct": 1184, |
| "total": 1196, |
| "rate": 0.9899665551839465 |
| }, |
| "tool_correct": { |
| "correct": 1077, |
| "total": 1100, |
| "rate": 0.9790909090909091 |
| }, |
| "arguments_exact": { |
| "correct": 753, |
| "total": 1100, |
| "rate": 0.6845454545454546 |
| }, |
| "sql_equivalent": { |
| "correct": 664, |
| "total": 912, |
| "rate": 0.7280701754385965 |
| }, |
| "success": { |
| "correct": 881, |
| "total": 1196, |
| "rate": 0.7366220735785953 |
| } |
| }, |
| "groups": { |
| "language:en": { |
| "json_valid": { |
| "correct": 299, |
| "total": 299, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 296, |
| "total": 299, |
| "rate": 0.9899665551839465 |
| }, |
| "action_correct": { |
| "correct": 293, |
| "total": 299, |
| "rate": 0.979933110367893 |
| }, |
| "tool_correct": { |
| "correct": 268, |
| "total": 275, |
| "rate": 0.9745454545454545 |
| }, |
| "arguments_exact": { |
| "correct": 169, |
| "total": 275, |
| "rate": 0.6145454545454545 |
| }, |
| "sql_equivalent": { |
| "correct": 146, |
| "total": 228, |
| "rate": 0.6403508771929824 |
| }, |
| "success": { |
| "correct": 197, |
| "total": 299, |
| "rate": 0.6588628762541806 |
| } |
| }, |
| "backend:mysql": { |
| "json_valid": { |
| "correct": 608, |
| "total": 608, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 608, |
| "total": 608, |
| "rate": 1.0 |
| }, |
| "action_correct": { |
| "correct": 601, |
| "total": 608, |
| "rate": 0.9884868421052632 |
| }, |
| "tool_correct": { |
| "correct": 543, |
| "total": 556, |
| "rate": 0.9766187050359713 |
| }, |
| "arguments_exact": { |
| "correct": 391, |
| "total": 556, |
| "rate": 0.7032374100719424 |
| }, |
| "sql_equivalent": { |
| "correct": 351, |
| "total": 476, |
| "rate": 0.7373949579831933 |
| }, |
| "success": { |
| "correct": 452, |
| "total": 608, |
| "rate": 0.743421052631579 |
| } |
| }, |
| "operation:all": { |
| "json_valid": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "action_correct": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "tool_correct": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "arguments_exact": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "sql_equivalent": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "success": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| } |
| }, |
| "language:noisy_en": { |
| "json_valid": { |
| "correct": 299, |
| "total": 299, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 299, |
| "total": 299, |
| "rate": 1.0 |
| }, |
| "action_correct": { |
| "correct": 293, |
| "total": 299, |
| "rate": 0.979933110367893 |
| }, |
| "tool_correct": { |
| "correct": 270, |
| "total": 275, |
| "rate": 0.9818181818181818 |
| }, |
| "arguments_exact": { |
| "correct": 195, |
| "total": 275, |
| "rate": 0.7090909090909091 |
| }, |
| "sql_equivalent": { |
| "correct": 166, |
| "total": 228, |
| "rate": 0.7280701754385965 |
| }, |
| "success": { |
| "correct": 221, |
| "total": 299, |
| "rate": 0.7391304347826086 |
| } |
| }, |
| "language:hi": { |
| "json_valid": { |
| "correct": 299, |
| "total": 299, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 299, |
| "total": 299, |
| "rate": 1.0 |
| }, |
| "action_correct": { |
| "correct": 299, |
| "total": 299, |
| "rate": 1.0 |
| }, |
| "tool_correct": { |
| "correct": 270, |
| "total": 275, |
| "rate": 0.9818181818181818 |
| }, |
| "arguments_exact": { |
| "correct": 183, |
| "total": 275, |
| "rate": 0.6654545454545454 |
| }, |
| "sql_equivalent": { |
| "correct": 162, |
| "total": 228, |
| "rate": 0.7105263157894737 |
| }, |
| "success": { |
| "correct": 222, |
| "total": 299, |
| "rate": 0.7424749163879598 |
| } |
| }, |
| "language:hinglish": { |
| "json_valid": { |
| "correct": 299, |
| "total": 299, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 293, |
| "total": 299, |
| "rate": 0.979933110367893 |
| }, |
| "action_correct": { |
| "correct": 299, |
| "total": 299, |
| "rate": 1.0 |
| }, |
| "tool_correct": { |
| "correct": 269, |
| "total": 275, |
| "rate": 0.9781818181818182 |
| }, |
| "arguments_exact": { |
| "correct": 206, |
| "total": 275, |
| "rate": 0.7490909090909091 |
| }, |
| "sql_equivalent": { |
| "correct": 190, |
| "total": 228, |
| "rate": 0.8333333333333334 |
| }, |
| "success": { |
| "correct": 241, |
| "total": 299, |
| "rate": 0.8060200668896321 |
| } |
| }, |
| "backend:supabase": { |
| "json_valid": { |
| "correct": 588, |
| "total": 588, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 579, |
| "total": 588, |
| "rate": 0.9846938775510204 |
| }, |
| "action_correct": { |
| "correct": 583, |
| "total": 588, |
| "rate": 0.9914965986394558 |
| }, |
| "tool_correct": { |
| "correct": 534, |
| "total": 544, |
| "rate": 0.9816176470588235 |
| }, |
| "arguments_exact": { |
| "correct": 362, |
| "total": 544, |
| "rate": 0.6654411764705882 |
| }, |
| "sql_equivalent": { |
| "correct": 313, |
| "total": 436, |
| "rate": 0.7178899082568807 |
| }, |
| "success": { |
| "correct": 429, |
| "total": 588, |
| "rate": 0.7295918367346939 |
| } |
| }, |
| "operation:project": { |
| "json_valid": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "action_correct": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "tool_correct": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "arguments_exact": { |
| "correct": 21, |
| "total": 28, |
| "rate": 0.75 |
| }, |
| "sql_equivalent": { |
| "correct": 21, |
| "total": 28, |
| "rate": 0.75 |
| }, |
| "success": { |
| "correct": 21, |
| "total": 28, |
| "rate": 0.75 |
| } |
| }, |
| "operation:eq": { |
| "json_valid": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "action_correct": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "tool_correct": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "arguments_exact": { |
| "correct": 13, |
| "total": 28, |
| "rate": 0.4642857142857143 |
| }, |
| "sql_equivalent": { |
| "correct": 14, |
| "total": 28, |
| "rate": 0.5 |
| }, |
| "success": { |
| "correct": 14, |
| "total": 28, |
| "rate": 0.5 |
| } |
| }, |
| "operation:project_eq": { |
| "json_valid": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "action_correct": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "tool_correct": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "arguments_exact": { |
| "correct": 26, |
| "total": 28, |
| "rate": 0.9285714285714286 |
| }, |
| "sql_equivalent": { |
| "correct": 26, |
| "total": 28, |
| "rate": 0.9285714285714286 |
| }, |
| "success": { |
| "correct": 26, |
| "total": 28, |
| "rate": 0.9285714285714286 |
| } |
| }, |
| "operation:gt": { |
| "json_valid": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "action_correct": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "tool_correct": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "arguments_exact": { |
| "correct": 3, |
| "total": 28, |
| "rate": 0.10714285714285714 |
| }, |
| "sql_equivalent": { |
| "correct": 3, |
| "total": 28, |
| "rate": 0.10714285714285714 |
| }, |
| "success": { |
| "correct": 3, |
| "total": 28, |
| "rate": 0.10714285714285714 |
| } |
| }, |
| "operation:lt": { |
| "json_valid": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "action_correct": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "tool_correct": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "arguments_exact": { |
| "correct": 27, |
| "total": 28, |
| "rate": 0.9642857142857143 |
| }, |
| "sql_equivalent": { |
| "correct": 27, |
| "total": 28, |
| "rate": 0.9642857142857143 |
| }, |
| "success": { |
| "correct": 27, |
| "total": 28, |
| "rate": 0.9642857142857143 |
| } |
| }, |
| "operation:gte": { |
| "json_valid": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "action_correct": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "tool_correct": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "arguments_exact": { |
| "correct": 11, |
| "total": 28, |
| "rate": 0.39285714285714285 |
| }, |
| "sql_equivalent": { |
| "correct": 11, |
| "total": 28, |
| "rate": 0.39285714285714285 |
| }, |
| "success": { |
| "correct": 11, |
| "total": 28, |
| "rate": 0.39285714285714285 |
| } |
| }, |
| "operation:lte": { |
| "json_valid": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "action_correct": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "tool_correct": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "arguments_exact": { |
| "correct": 13, |
| "total": 28, |
| "rate": 0.4642857142857143 |
| }, |
| "sql_equivalent": { |
| "correct": 13, |
| "total": 28, |
| "rate": 0.4642857142857143 |
| }, |
| "success": { |
| "correct": 13, |
| "total": 28, |
| "rate": 0.4642857142857143 |
| } |
| }, |
| "operation:between": { |
| "json_valid": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "action_correct": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "tool_correct": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "arguments_exact": { |
| "correct": 24, |
| "total": 28, |
| "rate": 0.8571428571428571 |
| }, |
| "sql_equivalent": { |
| "correct": 24, |
| "total": 28, |
| "rate": 0.8571428571428571 |
| }, |
| "success": { |
| "correct": 24, |
| "total": 28, |
| "rate": 0.8571428571428571 |
| } |
| }, |
| "operation:and": { |
| "json_valid": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "action_correct": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "tool_correct": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "arguments_exact": { |
| "correct": 16, |
| "total": 28, |
| "rate": 0.5714285714285714 |
| }, |
| "sql_equivalent": { |
| "correct": 16, |
| "total": 28, |
| "rate": 0.5714285714285714 |
| }, |
| "success": { |
| "correct": 16, |
| "total": 28, |
| "rate": 0.5714285714285714 |
| } |
| }, |
| "operation:or": { |
| "json_valid": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "action_correct": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "tool_correct": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "arguments_exact": { |
| "correct": 13, |
| "total": 28, |
| "rate": 0.4642857142857143 |
| }, |
| "sql_equivalent": { |
| "correct": 23, |
| "total": 28, |
| "rate": 0.8214285714285714 |
| }, |
| "success": { |
| "correct": 23, |
| "total": 28, |
| "rate": 0.8214285714285714 |
| } |
| }, |
| "operation:null": { |
| "json_valid": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "action_correct": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "tool_correct": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "arguments_exact": { |
| "correct": 18, |
| "total": 28, |
| "rate": 0.6428571428571429 |
| }, |
| "sql_equivalent": { |
| "correct": 18, |
| "total": 28, |
| "rate": 0.6428571428571429 |
| }, |
| "success": { |
| "correct": 18, |
| "total": 28, |
| "rate": 0.6428571428571429 |
| } |
| }, |
| "operation:not_null": { |
| "json_valid": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "action_correct": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "tool_correct": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "arguments_exact": { |
| "correct": 14, |
| "total": 28, |
| "rate": 0.5 |
| }, |
| "sql_equivalent": { |
| "correct": 21, |
| "total": 28, |
| "rate": 0.75 |
| }, |
| "success": { |
| "correct": 21, |
| "total": 28, |
| "rate": 0.75 |
| } |
| }, |
| "operation:count": { |
| "json_valid": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "action_correct": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "tool_correct": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "arguments_exact": { |
| "correct": 21, |
| "total": 28, |
| "rate": 0.75 |
| }, |
| "sql_equivalent": { |
| "correct": 21, |
| "total": 28, |
| "rate": 0.75 |
| }, |
| "success": { |
| "correct": 21, |
| "total": 28, |
| "rate": 0.75 |
| } |
| }, |
| "operation:count_eq": { |
| "json_valid": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "action_correct": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "tool_correct": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "arguments_exact": { |
| "correct": 13, |
| "total": 28, |
| "rate": 0.4642857142857143 |
| }, |
| "sql_equivalent": { |
| "correct": 13, |
| "total": 28, |
| "rate": 0.4642857142857143 |
| }, |
| "success": { |
| "correct": 13, |
| "total": 28, |
| "rate": 0.4642857142857143 |
| } |
| }, |
| "operation:sum": { |
| "json_valid": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "action_correct": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "tool_correct": { |
| "correct": 27, |
| "total": 28, |
| "rate": 0.9642857142857143 |
| }, |
| "arguments_exact": { |
| "correct": 21, |
| "total": 28, |
| "rate": 0.75 |
| }, |
| "sql_equivalent": { |
| "correct": 21, |
| "total": 28, |
| "rate": 0.75 |
| }, |
| "success": { |
| "correct": 21, |
| "total": 28, |
| "rate": 0.75 |
| } |
| }, |
| "operation:avg": { |
| "json_valid": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "action_correct": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "tool_correct": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "arguments_exact": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "sql_equivalent": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "success": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| } |
| }, |
| "operation:max": { |
| "json_valid": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "action_correct": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "tool_correct": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "arguments_exact": { |
| "correct": 16, |
| "total": 28, |
| "rate": 0.5714285714285714 |
| }, |
| "sql_equivalent": { |
| "correct": 16, |
| "total": 28, |
| "rate": 0.5714285714285714 |
| }, |
| "success": { |
| "correct": 16, |
| "total": 28, |
| "rate": 0.5714285714285714 |
| } |
| }, |
| "operation:min": { |
| "json_valid": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "action_correct": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "tool_correct": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "arguments_exact": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "sql_equivalent": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "success": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| } |
| }, |
| "operation:distinct": { |
| "json_valid": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "action_correct": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "tool_correct": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "arguments_exact": { |
| "correct": 21, |
| "total": 28, |
| "rate": 0.75 |
| }, |
| "sql_equivalent": { |
| "correct": 21, |
| "total": 28, |
| "rate": 0.75 |
| }, |
| "success": { |
| "correct": 21, |
| "total": 28, |
| "rate": 0.75 |
| } |
| }, |
| "operation:sort_desc": { |
| "json_valid": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "action_correct": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "tool_correct": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "arguments_exact": { |
| "correct": 12, |
| "total": 28, |
| "rate": 0.42857142857142855 |
| }, |
| "sql_equivalent": { |
| "correct": 12, |
| "total": 28, |
| "rate": 0.42857142857142855 |
| }, |
| "success": { |
| "correct": 12, |
| "total": 28, |
| "rate": 0.42857142857142855 |
| } |
| }, |
| "operation:sort_asc": { |
| "json_valid": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "action_correct": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "tool_correct": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "arguments_exact": { |
| "correct": 12, |
| "total": 28, |
| "rate": 0.42857142857142855 |
| }, |
| "sql_equivalent": { |
| "correct": 12, |
| "total": 28, |
| "rate": 0.42857142857142855 |
| }, |
| "success": { |
| "correct": 12, |
| "total": 28, |
| "rate": 0.42857142857142855 |
| } |
| }, |
| "operation:top": { |
| "json_valid": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "action_correct": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "tool_correct": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "arguments_exact": { |
| "correct": 8, |
| "total": 28, |
| "rate": 0.2857142857142857 |
| }, |
| "sql_equivalent": { |
| "correct": 8, |
| "total": 28, |
| "rate": 0.2857142857142857 |
| }, |
| "success": { |
| "correct": 8, |
| "total": 28, |
| "rate": 0.2857142857142857 |
| } |
| }, |
| "operation:bottom": { |
| "json_valid": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "action_correct": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "tool_correct": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "arguments_exact": { |
| "correct": 17, |
| "total": 28, |
| "rate": 0.6071428571428571 |
| }, |
| "sql_equivalent": { |
| "correct": 17, |
| "total": 28, |
| "rate": 0.6071428571428571 |
| }, |
| "success": { |
| "correct": 17, |
| "total": 28, |
| "rate": 0.6071428571428571 |
| } |
| }, |
| "operation:group_count": { |
| "json_valid": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "action_correct": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "tool_correct": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "arguments_exact": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "sql_equivalent": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "success": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| } |
| }, |
| "operation:group_sum": { |
| "json_valid": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "action_correct": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "tool_correct": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "arguments_exact": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "sql_equivalent": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "success": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| } |
| }, |
| "operation:having": { |
| "json_valid": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "action_correct": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "tool_correct": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "arguments_exact": { |
| "correct": 25, |
| "total": 28, |
| "rate": 0.8928571428571429 |
| }, |
| "sql_equivalent": { |
| "correct": 25, |
| "total": 28, |
| "rate": 0.8928571428571429 |
| }, |
| "success": { |
| "correct": 25, |
| "total": 28, |
| "rate": 0.8928571428571429 |
| } |
| }, |
| "operation:date_after": { |
| "json_valid": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "action_correct": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "tool_correct": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "arguments_exact": { |
| "correct": 19, |
| "total": 28, |
| "rate": 0.6785714285714286 |
| }, |
| "sql_equivalent": { |
| "correct": 22, |
| "total": 28, |
| "rate": 0.7857142857142857 |
| }, |
| "success": { |
| "correct": 22, |
| "total": 28, |
| "rate": 0.7857142857142857 |
| } |
| }, |
| "operation:year": { |
| "json_valid": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "action_correct": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "tool_correct": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "arguments_exact": { |
| "correct": 13, |
| "total": 28, |
| "rate": 0.4642857142857143 |
| }, |
| "sql_equivalent": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "success": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| } |
| }, |
| "operation:month": { |
| "json_valid": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "action_correct": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "tool_correct": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "arguments_exact": { |
| "correct": 12, |
| "total": 28, |
| "rate": 0.42857142857142855 |
| }, |
| "sql_equivalent": { |
| "correct": 26, |
| "total": 28, |
| "rate": 0.9285714285714286 |
| }, |
| "success": { |
| "correct": 26, |
| "total": 28, |
| "rate": 0.9285714285714286 |
| } |
| }, |
| "operation:contains": { |
| "json_valid": { |
| "correct": 24, |
| "total": 24, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 24, |
| "total": 24, |
| "rate": 1.0 |
| }, |
| "action_correct": { |
| "correct": 24, |
| "total": 24, |
| "rate": 1.0 |
| }, |
| "tool_correct": { |
| "correct": 24, |
| "total": 24, |
| "rate": 1.0 |
| }, |
| "arguments_exact": { |
| "correct": 19, |
| "total": 24, |
| "rate": 0.7916666666666666 |
| }, |
| "sql_equivalent": { |
| "correct": 19, |
| "total": 24, |
| "rate": 0.7916666666666666 |
| }, |
| "success": { |
| "correct": 19, |
| "total": 24, |
| "rate": 0.7916666666666666 |
| } |
| }, |
| "operation:join": { |
| "json_valid": { |
| "correct": 24, |
| "total": 24, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 24, |
| "total": 24, |
| "rate": 1.0 |
| }, |
| "action_correct": { |
| "correct": 24, |
| "total": 24, |
| "rate": 1.0 |
| }, |
| "tool_correct": { |
| "correct": 24, |
| "total": 24, |
| "rate": 1.0 |
| }, |
| "arguments_exact": { |
| "correct": 24, |
| "total": 24, |
| "rate": 1.0 |
| }, |
| "sql_equivalent": { |
| "correct": 24, |
| "total": 24, |
| "rate": 1.0 |
| }, |
| "success": { |
| "correct": 24, |
| "total": 24, |
| "rate": 1.0 |
| } |
| }, |
| "operation:join_filter": { |
| "json_valid": { |
| "correct": 24, |
| "total": 24, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 24, |
| "total": 24, |
| "rate": 1.0 |
| }, |
| "action_correct": { |
| "correct": 24, |
| "total": 24, |
| "rate": 1.0 |
| }, |
| "tool_correct": { |
| "correct": 24, |
| "total": 24, |
| "rate": 1.0 |
| }, |
| "arguments_exact": { |
| "correct": 22, |
| "total": 24, |
| "rate": 0.9166666666666666 |
| }, |
| "sql_equivalent": { |
| "correct": 22, |
| "total": 24, |
| "rate": 0.9166666666666666 |
| }, |
| "success": { |
| "correct": 22, |
| "total": 24, |
| "rate": 0.9166666666666666 |
| } |
| }, |
| "operation:list_tables": { |
| "json_valid": { |
| "correct": 24, |
| "total": 24, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 24, |
| "total": 24, |
| "rate": 1.0 |
| }, |
| "action_correct": { |
| "correct": 24, |
| "total": 24, |
| "rate": 1.0 |
| }, |
| "tool_correct": { |
| "correct": 24, |
| "total": 24, |
| "rate": 1.0 |
| }, |
| "arguments_exact": { |
| "correct": 24, |
| "total": 24, |
| "rate": 1.0 |
| }, |
| "sql_equivalent": { |
| "correct": 0, |
| "total": 0, |
| "rate": 0.0 |
| }, |
| "success": { |
| "correct": 24, |
| "total": 24, |
| "rate": 1.0 |
| } |
| }, |
| "operation:describe": { |
| "json_valid": { |
| "correct": 24, |
| "total": 24, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 21, |
| "total": 24, |
| "rate": 0.875 |
| }, |
| "action_correct": { |
| "correct": 24, |
| "total": 24, |
| "rate": 1.0 |
| }, |
| "tool_correct": { |
| "correct": 12, |
| "total": 24, |
| "rate": 0.5 |
| }, |
| "arguments_exact": { |
| "correct": 9, |
| "total": 24, |
| "rate": 0.375 |
| }, |
| "sql_equivalent": { |
| "correct": 0, |
| "total": 0, |
| "rate": 0.0 |
| }, |
| "success": { |
| "correct": 9, |
| "total": 24, |
| "rate": 0.375 |
| } |
| }, |
| "operation:missing_schema": { |
| "json_valid": { |
| "correct": 24, |
| "total": 24, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 24, |
| "total": 24, |
| "rate": 1.0 |
| }, |
| "action_correct": { |
| "correct": 24, |
| "total": 24, |
| "rate": 1.0 |
| }, |
| "tool_correct": { |
| "correct": 24, |
| "total": 24, |
| "rate": 1.0 |
| }, |
| "arguments_exact": { |
| "correct": 24, |
| "total": 24, |
| "rate": 1.0 |
| }, |
| "sql_equivalent": { |
| "correct": 0, |
| "total": 0, |
| "rate": 0.0 |
| }, |
| "success": { |
| "correct": 24, |
| "total": 24, |
| "rate": 1.0 |
| } |
| }, |
| "operation:sql_error": { |
| "json_valid": { |
| "correct": 20, |
| "total": 20, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 14, |
| "total": 20, |
| "rate": 0.7 |
| }, |
| "action_correct": { |
| "correct": 20, |
| "total": 20, |
| "rate": 1.0 |
| }, |
| "tool_correct": { |
| "correct": 10, |
| "total": 20, |
| "rate": 0.5 |
| }, |
| "arguments_exact": { |
| "correct": 6, |
| "total": 20, |
| "rate": 0.3 |
| }, |
| "sql_equivalent": { |
| "correct": 0, |
| "total": 0, |
| "rate": 0.0 |
| }, |
| "success": { |
| "correct": 6, |
| "total": 20, |
| "rate": 0.3 |
| } |
| }, |
| "operation:ambiguous": { |
| "json_valid": { |
| "correct": 24, |
| "total": 24, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 24, |
| "total": 24, |
| "rate": 1.0 |
| }, |
| "action_correct": { |
| "correct": 18, |
| "total": 24, |
| "rate": 0.75 |
| }, |
| "tool_correct": { |
| "correct": 0, |
| "total": 0, |
| "rate": 0.0 |
| }, |
| "arguments_exact": { |
| "correct": 0, |
| "total": 0, |
| "rate": 0.0 |
| }, |
| "sql_equivalent": { |
| "correct": 0, |
| "total": 0, |
| "rate": 0.0 |
| }, |
| "success": { |
| "correct": 18, |
| "total": 24, |
| "rate": 0.75 |
| } |
| }, |
| "operation:missing_value": { |
| "json_valid": { |
| "correct": 24, |
| "total": 24, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 24, |
| "total": 24, |
| "rate": 1.0 |
| }, |
| "action_correct": { |
| "correct": 24, |
| "total": 24, |
| "rate": 1.0 |
| }, |
| "tool_correct": { |
| "correct": 0, |
| "total": 0, |
| "rate": 0.0 |
| }, |
| "arguments_exact": { |
| "correct": 0, |
| "total": 0, |
| "rate": 0.0 |
| }, |
| "sql_equivalent": { |
| "correct": 0, |
| "total": 0, |
| "rate": 0.0 |
| }, |
| "success": { |
| "correct": 24, |
| "total": 24, |
| "rate": 1.0 |
| } |
| }, |
| "operation:write": { |
| "json_valid": { |
| "correct": 24, |
| "total": 24, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 24, |
| "total": 24, |
| "rate": 1.0 |
| }, |
| "action_correct": { |
| "correct": 18, |
| "total": 24, |
| "rate": 0.75 |
| }, |
| "tool_correct": { |
| "correct": 0, |
| "total": 0, |
| "rate": 0.0 |
| }, |
| "arguments_exact": { |
| "correct": 0, |
| "total": 0, |
| "rate": 0.0 |
| }, |
| "sql_equivalent": { |
| "correct": 0, |
| "total": 0, |
| "rate": 0.0 |
| }, |
| "success": { |
| "correct": 18, |
| "total": 24, |
| "rate": 0.75 |
| } |
| }, |
| "operation:weather": { |
| "json_valid": { |
| "correct": 24, |
| "total": 24, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 24, |
| "total": 24, |
| "rate": 1.0 |
| }, |
| "action_correct": { |
| "correct": 24, |
| "total": 24, |
| "rate": 1.0 |
| }, |
| "tool_correct": { |
| "correct": 24, |
| "total": 24, |
| "rate": 1.0 |
| }, |
| "arguments_exact": { |
| "correct": 24, |
| "total": 24, |
| "rate": 1.0 |
| }, |
| "sql_equivalent": { |
| "correct": 0, |
| "total": 0, |
| "rate": 0.0 |
| }, |
| "success": { |
| "correct": 24, |
| "total": 24, |
| "rate": 1.0 |
| } |
| }, |
| "operation:search": { |
| "json_valid": { |
| "correct": 24, |
| "total": 24, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 24, |
| "total": 24, |
| "rate": 1.0 |
| }, |
| "action_correct": { |
| "correct": 24, |
| "total": 24, |
| "rate": 1.0 |
| }, |
| "tool_correct": { |
| "correct": 24, |
| "total": 24, |
| "rate": 1.0 |
| }, |
| "arguments_exact": { |
| "correct": 24, |
| "total": 24, |
| "rate": 1.0 |
| }, |
| "sql_equivalent": { |
| "correct": 0, |
| "total": 0, |
| "rate": 0.0 |
| }, |
| "success": { |
| "correct": 24, |
| "total": 24, |
| "rate": 1.0 |
| } |
| }, |
| "operation:read_file": { |
| "json_valid": { |
| "correct": 24, |
| "total": 24, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 24, |
| "total": 24, |
| "rate": 1.0 |
| }, |
| "action_correct": { |
| "correct": 24, |
| "total": 24, |
| "rate": 1.0 |
| }, |
| "tool_correct": { |
| "correct": 24, |
| "total": 24, |
| "rate": 1.0 |
| }, |
| "arguments_exact": { |
| "correct": 24, |
| "total": 24, |
| "rate": 1.0 |
| }, |
| "sql_equivalent": { |
| "correct": 0, |
| "total": 0, |
| "rate": 0.0 |
| }, |
| "success": { |
| "correct": 24, |
| "total": 24, |
| "rate": 1.0 |
| } |
| }, |
| "operation:ticket": { |
| "json_valid": { |
| "correct": 24, |
| "total": 24, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 24, |
| "total": 24, |
| "rate": 1.0 |
| }, |
| "action_correct": { |
| "correct": 24, |
| "total": 24, |
| "rate": 1.0 |
| }, |
| "tool_correct": { |
| "correct": 24, |
| "total": 24, |
| "rate": 1.0 |
| }, |
| "arguments_exact": { |
| "correct": 4, |
| "total": 24, |
| "rate": 0.16666666666666666 |
| }, |
| "sql_equivalent": { |
| "correct": 0, |
| "total": 0, |
| "rate": 0.0 |
| }, |
| "success": { |
| "correct": 4, |
| "total": 24, |
| "rate": 0.16666666666666666 |
| } |
| }, |
| "operation:final": { |
| "json_valid": { |
| "correct": 24, |
| "total": 24, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 24, |
| "total": 24, |
| "rate": 1.0 |
| }, |
| "action_correct": { |
| "correct": 24, |
| "total": 24, |
| "rate": 1.0 |
| }, |
| "tool_correct": { |
| "correct": 0, |
| "total": 0, |
| "rate": 0.0 |
| }, |
| "arguments_exact": { |
| "correct": 0, |
| "total": 0, |
| "rate": 0.0 |
| }, |
| "sql_equivalent": { |
| "correct": 0, |
| "total": 0, |
| "rate": 0.0 |
| }, |
| "success": { |
| "correct": 18, |
| "total": 24, |
| "rate": 0.75 |
| } |
| } |
| } |
| } |
