evaluation/manual-gpu.jsonl

{"id": "manual_2ad3e251c936135e2005_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "all", "question": "all specimens pls", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM specimens;"}}, "output": "{\"action\":\"clarify\",\"question\":\"This connection is read-only. Would you like a SELECT query instead?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": false, "tool_correct": false, "arguments_exact": false, "sql_equivalent": false, "success": false}}
{"id": "manual_2e2ac3c5dcf95429daaa_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "null", "question": "specimens title null records", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM specimens WHERE title IS NULL;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens WHERE title IS NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "manual_0b8db53d711baed4f4ed_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "gte", "question": "specimens cost minimum 10 include same", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM specimens WHERE cost >= 10;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens ORDER BY cost ASC LIMIT 10;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}}
{"id": "manual_2ad3e251c936135e2005_en", "language": "en", "backend": "supabase", "operation": "all", "question": "I need everything in specimens.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "manual_efee65268e6dc2bb069e_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "project", "question": "specimens only customer_name show", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT customer_name FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT customer_name FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "manual_efee65268e6dc2bb069e_hinglish", "language": "hinglish", "backend": "supabase", "operation": "project", "question": "bas specimens ke customer_name dikha do", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT customer_name FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT customer_name FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "manual_0b8db53d711baed4f4ed_hinglish", "language": "hinglish", "backend": "supabase", "operation": "gte", "question": "specimens me cost kam se kam 10 ho", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM specimens WHERE cost >= 10;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens WHERE cost >= 10;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "manual_2e2ac3c5dcf95429daaa_hinglish", "language": "hinglish", "backend": "supabase", "operation": "null", "question": "specimens me jinka title NULL hai wo dikhao", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM specimens WHERE title IS NULL;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens WHERE price > 1500;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}}
{"id": "manual_2ad3e251c936135e2005_hinglish", "language": "hinglish", "backend": "supabase", "operation": "all", "question": "specimens ka sara data chahiye", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens WHERE display_name IS NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}}
{"id": "manual_d735c1ac1979ff1e8620_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "sum", "question": "inspections price all add", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT SUM(price) FROM inspections;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT SUM(price) FROM inspections;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "manual_0b8db53d711baed4f4ed_en", "language": "en", "backend": "supabase", "operation": "gte", "question": "Include specimens records at 10 or above in cost.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM specimens WHERE cost >= 10;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens WHERE cost >= 10;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "manual_2e2ac3c5dcf95429daaa_en", "language": "en", "backend": "supabase", "operation": "null", "question": "Find specimens records with no title value (NULL).", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM specimens WHERE title IS NULL;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens WHERE city = 'ApecULL';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}}
{"id": "manual_7dd163238936d96d0bd4_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "distinct", "question": "reservations city unique only", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT DISTINCT city FROM reservations;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT DISTINCT city FROM reservations;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "manual_efee65268e6dc2bb069e_en", "language": "en", "backend": "supabase", "operation": "project", "question": "Just the customer_name values from specimens, please.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT customer_name FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT customer_name FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "manual_eb982cf071094adaca4f_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "project_eq", "question": "reservations category Delhi only name", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT name FROM reservations WHERE category = 'Delhi';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT name FROM reservations WHERE category = 'Delhi';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "manual_d735c1ac1979ff1e8620_en", "language": "en", "backend": "supabase", "operation": "sum", "question": "Add up price across inspections.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT SUM(price) FROM inspections;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT display_name FROM inspections WHERE kind = 'Adddd';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}}
{"id": "manual_d735c1ac1979ff1e8620_hinglish", "language": "hinglish", "backend": "supabase", "operation": "sum", "question": "inspections ke price ka jod batao", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT SUM(price) FROM inspections;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT SUM(price) FROM inspections;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "manual_c4f25575abfd24e3cd63_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "lte", "question": "reservations cost max 1 equal also", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM reservations WHERE cost <= 1;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM reservations ORDER BY cost DESC LIMIT 1;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}}
{"id": "manual_7dd163238936d96d0bd4_hinglish", "language": "hinglish", "backend": "supabase", "operation": "distinct", "question": "reservations me city ki unique values kya hain", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT DISTINCT city FROM reservations;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT DISTINCT city FROM reservations;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "manual_eb982cf071094adaca4f_hinglish", "language": "hinglish", "backend": "supabase", "operation": "project_eq", "question": "reservations me category Delhi walon ka name batao", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT name FROM reservations WHERE category = 'Delhi';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT name FROM reservations WHERE category = 'Delhi';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "manual_eb982cf071094adaca4f_en", "language": "en", "backend": "supabase", "operation": "project_eq", "question": "In reservations, give me name for category Delhi.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT name FROM reservations WHERE category = 'Delhi';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT name FROM reservations WHERE category = 'Delhi';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "manual_c4f25575abfd24e3cd63_hinglish", "language": "hinglish", "backend": "supabase", "operation": "lte", "question": "reservations me cost 1 ya usse kam ho", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM reservations WHERE cost <= 1;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM reservations WHERE cost <= 1;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "manual_7dd163238936d96d0bd4_en", "language": "en", "backend": "supabase", "operation": "distinct", "question": "Which different city values occur in reservations?", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT DISTINCT city FROM reservations;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT DISTINCT city FROM reservations;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "manual_efee65268e6dc2bb069e_hi", "language": "hi", "backend": "supabase", "operation": "project", "question": "specimens से सिर्फ customer_name दिखाना।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT customer_name FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT customer_name FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "manual_0b8db53d711baed4f4ed_hi", "language": "hi", "backend": "supabase", "operation": "gte", "question": "specimens में cost कम से कम 10 होना चाहिए।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM specimens WHERE cost >= 10;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens WHERE cost >= 10;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "manual_d735c1ac1979ff1e8620_hi", "language": "hi", "backend": "supabase", "operation": "sum", "question": "inspections के price का जोड़ बताओ।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT SUM(price) FROM inspections;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT SUM(price) FROM inspections;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "manual_2ad3e251c936135e2005_hi", "language": "hi", "backend": "supabase", "operation": "all", "question": "specimens का सारा डेटा चाहिए।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "manual_c4f25575abfd24e3cd63_hi", "language": "hi", "backend": "supabase", "operation": "lte", "question": "reservations में cost 1 या उससे कम हो।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM reservations WHERE cost <= 1;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM reservations WHERE cost <= 1;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "manual_c4f25575abfd24e3cd63_en", "language": "en", "backend": "supabase", "operation": "lte", "question": "From reservations, include cost values up to and including 1.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM reservations WHERE cost <= 1;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM reservations WHERE cost BETWEEN 1 AND 1;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}}
{"id": "manual_eb982cf071094adaca4f_hi", "language": "hi", "backend": "supabase", "operation": "project_eq", "question": "reservations में category Delhi हो तो उनका name बताओ।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT name FROM reservations WHERE category = 'Delhi';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT name FROM reservations WHERE category = 'Delhi';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "manual_7dd163238936d96d0bd4_hi", "language": "hi", "backend": "supabase", "operation": "distinct", "question": "reservations में city के अलग-अलग मान कौन से हैं?", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT DISTINCT city FROM reservations;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT DISTINCT city FROM reservations;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "manual_2e2ac3c5dcf95429daaa_hi", "language": "hi", "backend": "supabase", "operation": "null", "question": "specimens में जिनका title NULL है वे रिकॉर्ड दिखाओ।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM specimens WHERE title IS NULL;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens WHERE price > 1500;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}}
{"id": "manual_5a8f2e207d0efe5b018c_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "list_tables", "question": "tables here list", "expected": {"action": "call", "name": "warehouse.list_tables", "arguments": {"project_id": "demo_specimens", "schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_specmemens\",\"query\":\"SELECT * FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": false, "arguments_exact": false, "sql_equivalent": null, "success": false}}
{"id": "manual_5a8f2e207d0efe5b018c_hinglish", "language": "hinglish", "backend": "supabase", "operation": "list_tables", "question": "yahan kaunsi tables hain", "expected": {"action": "call", "name": "warehouse.list_tables", "arguments": {"project_id": "demo_specimens", "schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"project_id\":\"demo_specmemens\",\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": null, "success": false}}
{"id": "manual_8ebc9e868ee558a8ddfe_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "ambiguous", "question": "specimens best ones", "expected": {"action": "clarify", "question": "What does best mean: which column and order?"}, "output": "{\"action\":\"clarify\",\"question\":\"What does best mean: which column and order?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}}
{"id": "manual_35406fa024e66bcb5dd7_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "lt", "question": "specimens score less 10000", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_specimens", "query": "SELECT * FROM specimens WHERE score < 10000;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_specimens\",\"query\":\"SELECT * FROM specimens WHERE score < 10000;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "manual_82c104614df1ce6e2762_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "gt", "question": "specimens salary more than 100", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_specimens", "query": "SELECT * FROM specimens WHERE salary > 100;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_specimens\",\"query\":\"SELECT * FROM specimens WHERE salary > 100;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "manual_82c104614df1ce6e2762_en", "language": "en", "backend": "supabase", "operation": "gt", "question": "Which specimens entries have salary above 100?", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_specimens", "query": "SELECT * FROM specimens WHERE salary > 100;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_specmemens\",\"query\":\"SELECT * FROM specimens WHERE salary > 100;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": true, "success": false}}
{"id": "manual_35406fa024e66bcb5dd7_en", "language": "en", "backend": "supabase", "operation": "lt", "question": "Find specimens records below 10000 in score.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_specimens", "query": "SELECT * FROM specimens WHERE score < 10000;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_specmemens\",\"query\":\"SELECT * FROM specimens WHERE score < 10000;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": true, "success": false}}
{"id": "manual_b564e1a7b1e7fc6d2ab4_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "avg", "question": "specimens avg score tell", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_specimens", "query": "SELECT AVG(score) FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_specimens\",\"query\":\"SELECT AVG(score) FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "manual_5a8f2e207d0efe5b018c_en", "language": "en", "backend": "supabase", "operation": "list_tables", "question": "What tables can I query here?", "expected": {"action": "call", "name": "warehouse.list_tables", "arguments": {"project_id": "demo_specimens", "schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"project_id\":\"demo_specmens\",\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": null, "success": false}}
{"id": "manual_8ebc9e868ee558a8ddfe_en", "language": "en", "backend": "supabase", "operation": "ambiguous", "question": "Pick the best entries in specimens.", "expected": {"action": "clarify", "question": "What does best mean: which column and order?"}, "output": "{\"action\":\"clarify\",\"question\":\"What does best mean: which column and order?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}}
{"id": "manual_8ebc9e868ee558a8ddfe_hinglish", "language": "hinglish", "backend": "supabase", "operation": "ambiguous", "question": "specimens me best records chun lo", "expected": {"action": "clarify", "question": "Best ka matlab kya hai? Kaunsa column aur order?"}, "output": "{\"action\":\"clarify\",\"question\":\"Best ka matlab kya hai? Kaunsa column aur order?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}}
{"id": "manual_35406fa024e66bcb5dd7_hinglish", "language": "hinglish", "backend": "supabase", "operation": "lt", "question": "specimens me score 10000 se kam wale lao", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_specimens", "query": "SELECT * FROM specimens WHERE score < 10000;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_specimens\",\"query\":\"SELECT * FROM specimens WHERE score < 10000;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "manual_82c104614df1ce6e2762_hinglish", "language": "hinglish", "backend": "supabase", "operation": "gt", "question": "specimens me salary 100 se upar wale kaun hain", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_specimens", "query": "SELECT * FROM specimens WHERE salary > 100;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_specimens\",\"query\":\"SELECT * FROM specimens WHERE salary > 100;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "manual_b564e1a7b1e7fc6d2ab4_en", "language": "en", "backend": "supabase", "operation": "avg", "question": "What does score average out to in specimens?", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_specimens", "query": "SELECT AVG(score) FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_specmemens\",\"query\":\"SELECT AVG(score) FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": true, "success": false}}
{"id": "manual_b564e1a7b1e7fc6d2ab4_hinglish", "language": "hinglish", "backend": "supabase", "operation": "avg", "question": "specimens me score ka average kya hai", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_specimens", "query": "SELECT AVG(score) FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_specimens\",\"query\":\"SELECT AVG(score) FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "manual_40464b00088c9159e3d0_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "count", "question": "inspections total rows how much", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_inspections", "query": "SELECT COUNT(*) FROM inspections;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_inspections\",\"query\":\"SELECT COUNT(*) FROM inspections;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "manual_475287613764159ee5eb_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "group_count", "question": "reservations count per category", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_reservations", "query": "SELECT category, COUNT(*) FROM reservations GROUP BY category;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_reservations\",\"query\":\"SELECT category, COUNT(*) FROM reservations GROUP BY category;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "manual_fd0bab1cc08d7298f131_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "top", "question": "exhibits top 20 by salary big first", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_exhibits", "query": "SELECT * FROM exhibits ORDER BY salary DESC LIMIT 20;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_exhibits\",\"query\":\"SELECT * FROM exhibits ORDER BY salary DESC LIMIT 20;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "manual_40464b00088c9159e3d0_hinglish", "language": "hinglish", "backend": "supabase", "operation": "count", "question": "inspections me total kitni rows hain", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_inspections", "query": "SELECT COUNT(*) FROM inspections;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_inspections\",\"query\":\"SELECT COUNT(*) FROM inspections;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "manual_40464b00088c9159e3d0_en", "language": "en", "backend": "supabase", "operation": "count", "question": "How big is inspections, measured in rows?", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_inspections", "query": "SELECT COUNT(*) FROM inspections;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_inspections\",\"query\":\"SELECT COUNT(*) FROM inspections;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "manual_fd0bab1cc08d7298f131_hinglish", "language": "hinglish", "backend": "supabase", "operation": "top", "question": "exhibits me sabse bade salary wali 20 rows descending dikhao", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_exhibits", "query": "SELECT * FROM exhibits ORDER BY salary DESC LIMIT 20;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_exhibits\",\"query\":\"SELECT * FROM exhibits ORDER BY salary DESC LIMIT 20;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "manual_475287613764159ee5eb_hinglish", "language": "hinglish", "backend": "supabase", "operation": "group_count", "question": "reservations me har category ki count alag batao", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_reservations", "query": "SELECT category, COUNT(*) FROM reservations GROUP BY category;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_reservations\",\"query\":\"SELECT category, COUNT(*) FROM reservations GROUP BY category;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "manual_5a8f2e207d0efe5b018c_hi", "language": "hi", "backend": "supabase", "operation": "list_tables", "question": "यहाँ कौन-कौन सी टेबल हैं?", "expected": {"action": "call", "name": "warehouse.list_tables", "arguments": {"project_id": "demo_specimens", "schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_specmemens\",\"query\":\"SELECT * FROM specimens WHERE kind = 'Delhi';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": false, "arguments_exact": false, "sql_equivalent": null, "success": false}}
{"id": "manual_b564e1a7b1e7fc6d2ab4_hi", "language": "hi", "backend": "supabase", "operation": "avg", "question": "specimens में score का औसत क्या है?", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_specimens", "query": "SELECT AVG(score) FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_specimens\",\"query\":\"SELECT AVG(score) FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "manual_fd0bab1cc08d7298f131_en", "language": "en", "backend": "supabase", "operation": "top", "question": "Give me 20 entries in exhibits with the largest salary, largest first.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_exhibits", "query": "SELECT * FROM exhibits ORDER BY salary DESC LIMIT 20;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_exhibits\",\"query\":\"SELECT * FROM exhibits ORDER BY salary DESC LIMIT 20;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "manual_475287613764159ee5eb_en", "language": "en", "backend": "supabase", "operation": "group_count", "question": "Break down the number of reservations rows by category.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_reservations", "query": "SELECT category, COUNT(*) FROM reservations GROUP BY category;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_reservations\",\"query\":\"SELECT COUNT(*) FROM reservations WHERE category = 'Areak downs';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}}
{"id": "manual_82c104614df1ce6e2762_hi", "language": "hi", "backend": "supabase", "operation": "gt", "question": "specimens में किनका salary 100 से ज़्यादा है?", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_specimens", "query": "SELECT * FROM specimens WHERE salary > 100;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_specimens\",\"query\":\"SELECT * FROM specimens WHERE salary > 100;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "manual_8ebc9e868ee558a8ddfe_hi", "language": "hi", "backend": "supabase", "operation": "ambiguous", "question": "specimens में सबसे अच्छे रिकॉर्ड चुनो।", "expected": {"action": "clarify", "question": "सबसे अच्छा से आपका क्या मतलब है? कौन सा कॉलम और क्रम?"}, "output": "{\"action\":\"clarify\",\"question\":\"सबसे अच्छा से आपका क्या मतलब है? कौन सा कॉलम और क्रम?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}}
{"id": "manual_c4210296305105ab7472_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "ambiguous", "question": "specimens best ones", "expected": {"action": "clarify", "question": "What does best mean: which column and order?"}, "output": "{\"action\":\"clarify\",\"question\":\"What does best mean: which column and order?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}}
{"id": "manual_9bf8948becf74e81a103_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "list_tables", "question": "tables here list", "expected": {"action": "call", "name": "warehouse.list_tables", "arguments": {"schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": false, "arguments_exact": false, "sql_equivalent": null, "success": false}}
{"id": "manual_9bf8948becf74e81a103_hinglish", "language": "hinglish", "backend": "mysql", "operation": "list_tables", "question": "yahan kaunsi tables hain", "expected": {"action": "call", "name": "warehouse.list_tables", "arguments": {"schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "manual_8efcf517aee64807a8e5_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "avg", "question": "specimens avg balance tell", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT AVG(balance) FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT AVG(balance) FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "manual_adff4eabe2639c0c128c_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "all", "question": "all inspections pls", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"query": "SELECT * FROM inspections;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT * FROM inspections WHERE description IS NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}}
{"id": "manual_35406fa024e66bcb5dd7_hi", "language": "hi", "backend": "supabase", "operation": "lt", "question": "specimens में score 10000 से कम वाले रिकॉर्ड लाओ।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_specimens", "query": "SELECT * FROM specimens WHERE score < 10000;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_specimens\",\"query\":\"SELECT * FROM specimens WHERE score < 10000;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "manual_23c0aa3a4596f995ad2a_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "count", "question": "specimens total rows how much", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"query": "SELECT COUNT(*) FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT COUNT(*) FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "manual_475287613764159ee5eb_hi", "language": "hi", "backend": "supabase", "operation": "group_count", "question": "reservations में हर category की गिनती अलग-अलग बताओ।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_reservations", "query": "SELECT category, COUNT(*) FROM reservations GROUP BY category;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_reservations\",\"query\":\"SELECT category, COUNT(*) FROM reservations GROUP BY category;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "manual_23c0aa3a4596f995ad2a_hinglish", "language": "hinglish", "backend": "mysql", "operation": "count", "question": "specimens me total kitni rows hain", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"query": "SELECT COUNT(*) FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT COUNT(*) FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "manual_2164b376f732a1c371ab_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "group_count", "question": "specimens count per kind", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"sql": "SELECT kind, COUNT(*) FROM specimens GROUP BY kind;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT kind, COUNT(*) FROM specimens GROUP BY kind;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "manual_40464b00088c9159e3d0_hi", "language": "hi", "backend": "supabase", "operation": "count", "question": "inspections में कुल कितनी पंक्तियाँ हैं?", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_inspections", "query": "SELECT COUNT(*) FROM inspections;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_inspections\",\"query\":\"SELECT COUNT(*) FROM inspections;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "manual_8efcf517aee64807a8e5_en", "language": "en", "backend": "mysql", "operation": "avg", "question": "What does balance average out to in specimens?", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT AVG(balance) FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT AVG(balance) FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "manual_8efcf517aee64807a8e5_hinglish", "language": "hinglish", "backend": "mysql", "operation": "avg", "question": "specimens me balance ka average kya hai", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT AVG(balance) FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT AVG(balance) FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "manual_c4210296305105ab7472_en", "language": "en", "backend": "mysql", "operation": "ambiguous", "question": "Pick the best entries in specimens.", "expected": {"action": "clarify", "question": "What does best mean: which column and order?"}, "output": "{\"action\":\"clarify\",\"question\":\"What does best mean: which column and order?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}}
{"id": "manual_c4210296305105ab7472_hinglish", "language": "hinglish", "backend": "mysql", "operation": "ambiguous", "question": "specimens me best records chun lo", "expected": {"action": "clarify", "question": "Best ka matlab kya hai? Kaunsa column aur order?"}, "output": "{\"action\":\"clarify\",\"question\":\"Best ka matlab kya hai? Kaunsa column aur order?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}}
{"id": "manual_23c0aa3a4596f995ad2a_en", "language": "en", "backend": "mysql", "operation": "count", "question": "How big is specimens, measured in rows?", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"query": "SELECT COUNT(*) FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT COUNT(*) FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "manual_d30264bbc3b08aedbc51_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "sum", "question": "exhibits score all add", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"query": "SELECT SUM(score) FROM exhibits;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT SUM(score) FROM exhibits;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "manual_9bf8948becf74e81a103_en", "language": "en", "backend": "mysql", "operation": "list_tables", "question": "What tables can I query here?", "expected": {"action": "call", "name": "warehouse.list_tables", "arguments": {"schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": false, "action_correct": true, "tool_correct": false, "arguments_exact": false, "sql_equivalent": null, "success": false, "error": "'sql' is a required property\n\nFailed validating 'required' in schema:\n    {'type': 'object',\n     'properties': {'sql': {'type': 'string'}},\n     'required': ['sql'],\n     'additionalProperties': False}\n\nOn instance:\n    {'schemas': ['public']}"}}
{"id": "manual_adff4eabe2639c0c128c_en", "language": "en", "backend": "mysql", "operation": "all", "question": "I need everything in inspections.", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"query": "SELECT * FROM inspections;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT * FROM inspections;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "manual_b9c176c07809f42564a0_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "top", "question": "inspections top 5 by amount big first", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"query": "SELECT * FROM inspections ORDER BY amount DESC LIMIT 5;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT * FROM inspections ORDER BY amount DESC LIMIT 5;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "manual_c023a5b6f95173ae856d_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "gt", "question": "exhibits score more than 50", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"query": "SELECT * FROM exhibits WHERE score > 50;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT * FROM exhibits WHERE score > 50;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "manual_adff4eabe2639c0c128c_hinglish", "language": "hinglish", "backend": "mysql", "operation": "all", "question": "inspections ka sara data chahiye", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"query": "SELECT * FROM inspections;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT * FROM inspections;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "manual_cab55a7429427fbc5093_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "distinct", "question": "inspections department unique only", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"query": "SELECT DISTINCT department FROM inspections;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT DISTINCT department FROM inspections;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "manual_e8f90532d34c12209a83_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "gte", "question": "inspections salary minimum 1500 include same", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM inspections WHERE salary >= 1500;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM inspections WHERE salary >= 1500;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "manual_c023a5b6f95173ae856d_en", "language": "en", "backend": "mysql", "operation": "gt", "question": "Which exhibits entries have score above 50?", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"query": "SELECT * FROM exhibits WHERE score > 50;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT * FROM exhibits WHERE score > 50;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "manual_affa5e5e14444cd82bb4_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "lt", "question": "reservations salary less 10", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT * FROM reservations WHERE salary < 10;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM reservations WHERE salary < 10;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "manual_2164b376f732a1c371ab_hinglish", "language": "hinglish", "backend": "mysql", "operation": "group_count", "question": "specimens me har kind ki count alag batao", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"sql": "SELECT kind, COUNT(*) FROM specimens GROUP BY kind;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT kind, COUNT(*) FROM specimens GROUP BY kind;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "manual_d30264bbc3b08aedbc51_en", "language": "en", "backend": "mysql", "operation": "sum", "question": "Add up score across exhibits.", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"query": "SELECT SUM(score) FROM exhibits;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT name FROM exhibits WHERE city = 'Adddd';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false, "error": "no such column: name"}}
{"id": "manual_d30264bbc3b08aedbc51_hinglish", "language": "hinglish", "backend": "mysql", "operation": "sum", "question": "exhibits ke score ka jod batao", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"query": "SELECT SUM(score) FROM exhibits;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT SUM(score) FROM exhibits;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "manual_b02f421828127e2eb96b_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "lte", "question": "exhibits price max 2 equal also", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"sql": "SELECT * FROM exhibits WHERE price <= 2;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM exhibits ORDER BY price DESC LIMIT 2;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}}
{"id": "manual_b9c176c07809f42564a0_hinglish", "language": "hinglish", "backend": "mysql", "operation": "top", "question": "inspections me sabse bade amount wali 5 rows descending dikhao", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"query": "SELECT * FROM inspections ORDER BY amount DESC LIMIT 5;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT * FROM inspections ORDER BY amount DESC LIMIT 5;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "manual_e8f90532d34c12209a83_hinglish", "language": "hinglish", "backend": "mysql", "operation": "gte", "question": "inspections me salary kam se kam 1500 ho", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM inspections WHERE salary >= 1500;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM inspections WHERE salary >= 1500;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "manual_2c31add7d7b45843dec1_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "project_eq", "question": "inspections kind Pune only display_name", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT display_name FROM inspections WHERE kind = 'Pune';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT display_name FROM inspections WHERE kind = 'Pune';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "manual_c023a5b6f95173ae856d_hinglish", "language": "hinglish", "backend": "mysql", "operation": "gt", "question": "exhibits me score 50 se upar wale kaun hain", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"query": "SELECT * FROM exhibits WHERE score > 50;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT * FROM exhibits WHERE score > 50;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "manual_2164b376f732a1c371ab_en", "language": "en", "backend": "mysql", "operation": "group_count", "question": "Break down the number of specimens rows by kind.", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"sql": "SELECT kind, COUNT(*) FROM specimens GROUP BY kind;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT COUNT(*) FROM specimens WHERE kind = 'ureak down';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}}
{"id": "manual_b02f421828127e2eb96b_hinglish", "language": "hinglish", "backend": "mysql", "operation": "lte", "question": "exhibits me price 2 ya usse kam ho", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"sql": "SELECT * FROM exhibits WHERE price <= 2;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM exhibits WHERE price <= 2;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "manual_8efcf517aee64807a8e5_hi", "language": "hi", "backend": "mysql", "operation": "avg", "question": "specimens में balance का औसत क्या है?", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT AVG(balance) FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT AVG(balance) FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "manual_cab55a7429427fbc5093_hinglish", "language": "hinglish", "backend": "mysql", "operation": "distinct", "question": "inspections me department ki unique values kya hain", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"query": "SELECT DISTINCT department FROM inspections;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT DISTINCT department FROM inspections;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "manual_b9c176c07809f42564a0_en", "language": "en", "backend": "mysql", "operation": "top", "question": "Give me 5 entries in inspections with the largest amount, largest first.", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"query": "SELECT * FROM inspections ORDER BY amount DESC LIMIT 5;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT * FROM inspections ORDER BY amount DESC LIMIT 5;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "manual_a060620c638d2dc3759e_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "project", "question": "reservations only item_name show", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT item_name FROM reservations;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT item_name FROM reservations;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "manual_affa5e5e14444cd82bb4_en", "language": "en", "backend": "mysql", "operation": "lt", "question": "Find reservations records below 10 in salary.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT * FROM reservations WHERE salary < 10;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM reservations WHERE salary < 10;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "manual_affa5e5e14444cd82bb4_hinglish", "language": "hinglish", "backend": "mysql", "operation": "lt", "question": "reservations me salary 10 se kam wale lao", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT * FROM reservations WHERE salary < 10;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM reservations WHERE salary < 10;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "manual_e8f90532d34c12209a83_en", "language": "en", "backend": "mysql", "operation": "gte", "question": "Include inspections records at 1500 or above in salary.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM inspections WHERE salary >= 1500;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM inspections WHERE salary >= 1500;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "manual_04b2e5886e750df09a51_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "null", "question": "reservations display_name null records", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"sql": "SELECT * FROM reservations WHERE display_name IS NULL;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM reservations WHERE display_name IS NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "manual_9bf8948becf74e81a103_hi", "language": "hi", "backend": "mysql", "operation": "list_tables", "question": "यहाँ कौन-कौन सी टेबल हैं?", "expected": {"action": "call", "name": "warehouse.list_tables", "arguments": {"schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM specimens WHERE kind = 'Jaipur';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": false, "arguments_exact": false, "sql_equivalent": null, "success": false}}
{"id": "manual_cab55a7429427fbc5093_en", "language": "en", "backend": "mysql", "operation": "distinct", "question": "Which different department values occur in inspections?", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"query": "SELECT DISTINCT department FROM inspections;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT DISTINCT department FROM inspections;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "manual_2c31add7d7b45843dec1_en", "language": "en", "backend": "mysql", "operation": "project_eq", "question": "In inspections, give me display_name for kind Pune.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT display_name FROM inspections WHERE kind = 'Pune';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT display_name FROM inspections WHERE kind = 'Pune';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "manual_2c31add7d7b45843dec1_hinglish", "language": "hinglish", "backend": "mysql", "operation": "project_eq", "question": "inspections me kind Pune walon ka display_name batao", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT display_name FROM inspections WHERE kind = 'Pune';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT display_name FROM inspections WHERE kind = 'Pune';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "manual_a060620c638d2dc3759e_hinglish", "language": "hinglish", "backend": "mysql", "operation": "project", "question": "bas reservations ke item_name dikha do", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT item_name FROM reservations;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT item_name FROM reservations;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "manual_b02f421828127e2eb96b_en", "language": "en", "backend": "mysql", "operation": "lte", "question": "From exhibits, include price values up to and including 2.", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"sql": "SELECT * FROM exhibits WHERE price <= 2;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM exhibits WHERE price BETWEEN 2 AND 2;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}}
{"id": "manual_b02f421828127e2eb96b_hi", "language": "hi", "backend": "mysql", "operation": "lte", "question": "exhibits में price 2 या उससे कम हो।", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"sql": "SELECT * FROM exhibits WHERE price <= 2;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM exhibits WHERE price <= 2;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "manual_d30264bbc3b08aedbc51_hi", "language": "hi", "backend": "mysql", "operation": "sum", "question": "exhibits के score का जोड़ बताओ।", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"query": "SELECT SUM(score) FROM exhibits;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT SUM(score) FROM exhibits;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "manual_04b2e5886e750df09a51_hinglish", "language": "hinglish", "backend": "mysql", "operation": "null", "question": "reservations me jinka display_name NULL hai wo dikhao", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"sql": "SELECT * FROM reservations WHERE display_name IS NULL;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM reservations WHERE quantity > 1500;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}}
{"id": "manual_a060620c638d2dc3759e_en", "language": "en", "backend": "mysql", "operation": "project", "question": "Just the item_name values from reservations, please.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT item_name FROM reservations;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT item_name FROM reservations;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "manual_adff4eabe2639c0c128c_hi", "language": "hi", "backend": "mysql", "operation": "all", "question": "inspections का सारा डेटा चाहिए।", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"query": "SELECT * FROM inspections;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT * FROM inspections;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "manual_c4210296305105ab7472_hi", "language": "hi", "backend": "mysql", "operation": "ambiguous", "question": "specimens में सबसे अच्छे रिकॉर्ड चुनो।", "expected": {"action": "clarify", "question": "सबसे अच्छा से आपका क्या मतलब है? कौन सा कॉलम और क्रम?"}, "output": "{\"action\":\"clarify\",\"question\":\"सबसे अच्छा से आपका क्या मतलब है? कौन सा कॉलम और क्रम?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}}
{"id": "manual_23c0aa3a4596f995ad2a_hi", "language": "hi", "backend": "mysql", "operation": "count", "question": "specimens में कुल कितनी पंक्तियाँ हैं?", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"query": "SELECT COUNT(*) FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT COUNT(*) FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "manual_2164b376f732a1c371ab_hi", "language": "hi", "backend": "mysql", "operation": "group_count", "question": "specimens में हर kind की गिनती अलग-अलग बताओ।", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"sql": "SELECT kind, COUNT(*) FROM specimens GROUP BY kind;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT kind, COUNT(*) FROM specimens GROUP BY kind;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "manual_c023a5b6f95173ae856d_hi", "language": "hi", "backend": "mysql", "operation": "gt", "question": "exhibits में किनका score 50 से ज़्यादा है?", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"query": "SELECT * FROM exhibits WHERE score > 50;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT * FROM exhibits WHERE score > 50;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "manual_e8f90532d34c12209a83_hi", "language": "hi", "backend": "mysql", "operation": "gte", "question": "inspections में salary कम से कम 1500 होना चाहिए।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM inspections WHERE salary >= 1500;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM inspections WHERE salary >= 1500;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "manual_04b2e5886e750df09a51_en", "language": "en", "backend": "mysql", "operation": "null", "question": "Find reservations records with no display_name value (NULL).", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"sql": "SELECT * FROM reservations WHERE display_name IS NULL;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM reservations WHERE display_name IS NOT NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}}
{"id": "manual_2c31add7d7b45843dec1_hi", "language": "hi", "backend": "mysql", "operation": "project_eq", "question": "inspections में kind Pune हो तो उनका display_name बताओ।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT display_name FROM inspections WHERE kind = 'Pune';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT display_name FROM inspections WHERE kind = 'Pune';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "manual_fd0bab1cc08d7298f131_hi", "language": "hi", "backend": "supabase", "operation": "top", "question": "exhibits में सबसे ज़्यादा salary वाली 20 पंक्तियाँ घटते क्रम में दिखाओ।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_exhibits", "query": "SELECT * FROM exhibits ORDER BY salary DESC LIMIT 20;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_exhibits\",\"query\":\"SELECT * FROM exhibits ORDER BY salary DESC LIMIT 20;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "manual_cab55a7429427fbc5093_hi", "language": "hi", "backend": "mysql", "operation": "distinct", "question": "inspections में department के अलग-अलग मान कौन से हैं?", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"query": "SELECT DISTINCT department FROM inspections;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT DISTINCT department FROM inspections;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "manual_a060620c638d2dc3759e_hi", "language": "hi", "backend": "mysql", "operation": "project", "question": "reservations से सिर्फ item_name दिखाना।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT item_name FROM reservations;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT item_name FROM reservations;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "manual_affa5e5e14444cd82bb4_hi", "language": "hi", "backend": "mysql", "operation": "lt", "question": "reservations में salary 10 से कम वाले रिकॉर्ड लाओ।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT * FROM reservations WHERE salary < 10;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM reservations WHERE salary < 10;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "manual_04b2e5886e750df09a51_hi", "language": "hi", "backend": "mysql", "operation": "null", "question": "reservations में जिनका display_name NULL है वे रिकॉर्ड दिखाओ।", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"sql": "SELECT * FROM reservations WHERE display_name IS NULL;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM reservations WHERE quantity > 1500;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}}
{"id": "manual_b9c176c07809f42564a0_hi", "language": "hi", "backend": "mysql", "operation": "top", "question": "inspections में सबसे ज़्यादा amount वाली 5 पंक्तियाँ घटते क्रम में दिखाओ।", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"query": "SELECT * FROM inspections ORDER BY amount DESC LIMIT 5;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT * FROM inspections ORDER BY amount DESC LIMIT 5;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "manual_88b478ce80eed61bdfb4_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "weather", "question": "Jaipur weather celsius pls", "expected": {"action": "call", "name": "external.get_weather", "arguments": {"city": "Jaipur", "unit": "celsius"}}, "output": "{\"action\":\"call\",\"name\":\"external.get_weather\",\"arguments\":{\"city\":\"Jaipur\",\"unit\":\"celsius\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "manual_88b478ce80eed61bdfb4_hinglish", "language": "hinglish", "backend": "supabase", "operation": "weather", "question": "Jaipur ka weather celsius me batao", "expected": {"action": "call", "name": "external.get_weather", "arguments": {"city": "Jaipur", "unit": "celsius"}}, "output": "{\"action\":\"call\",\"name\":\"external.get_weather\",\"arguments\":{\"city\":\"Jaipur\",\"unit\":\"celsius\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "manual_88b478ce80eed61bdfb4_en", "language": "en", "backend": "supabase", "operation": "weather", "question": "What is the weather like in Jaipur? Use celsius.", "expected": {"action": "call", "name": "external.get_weather", "arguments": {"city": "Jaipur", "unit": "celsius"}}, "output": "{\"action\":\"call\",\"name\":\"external.get_weather\",\"arguments\":{\"city\":\"Jaipur\",\"unit\":\"celsius\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "manual_88b478ce80eed61bdfb4_hi", "language": "hi", "backend": "supabase", "operation": "weather", "question": "Jaipur का मौसम celsius में बताओ।", "expected": {"action": "call", "name": "external.get_weather", "arguments": {"city": "Jaipur", "unit": "celsius"}}, "output": "{\"action\":\"call\",\"name\":\"external.get_weather\",\"arguments\":{\"city\":\"Jaipur\",\"unit\":\"celsius\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "manual_4e42bbd40467b2b8c27f_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "search", "question": "docs find SQL joins", "expected": {"action": "call", "name": "external.search_docs", "arguments": {"query": "SQL joins"}}, "output": "{\"action\":\"call\",\"name\":\"external.search_docs\",\"arguments\":{\"query\":\"SQL joins\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "manual_8a696e3c3522dd69d5ac_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "ticket", "question": "ticket TKT-4486 details", "expected": {"action": "call", "name": "external.get_ticket", "arguments": {"ticket_id": "TKT-4486"}}, "output": "{\"action\":\"call\",\"name\":\"external.get_ticket\",\"arguments\":{\"ticket_id\":\"TKT-4486\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "manual_4e42bbd40467b2b8c27f_en", "language": "en", "backend": "supabase", "operation": "search", "question": "Find documentation about SQL joins.", "expected": {"action": "call", "name": "external.search_docs", "arguments": {"query": "SQL joins"}}, "output": "{\"action\":\"call\",\"name\":\"external.search_docs\",\"arguments\":{\"query\":\"SQL joins\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "manual_f73e21e0eb10d6617995_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "read_file", "question": "/config/app.json read pls", "expected": {"action": "call", "name": "external.read_file", "arguments": {"path": "/config/app.json"}}, "output": "{\"action\":\"call\",\"name\":\"external.read_file\",\"arguments\":{\"path\":\"/config/app.json\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "manual_8a696e3c3522dd69d5ac_hinglish", "language": "hinglish", "backend": "supabase", "operation": "ticket", "question": "ticket TKT-4486 ki details lao", "expected": {"action": "call", "name": "external.get_ticket", "arguments": {"ticket_id": "TKT-4486"}}, "output": "{\"action\":\"call\",\"name\":\"external.get_ticket\",\"arguments\":{\"ticket_id\":\"TKT-4486\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "manual_4e42bbd40467b2b8c27f_hinglish", "language": "hinglish", "backend": "supabase", "operation": "search", "question": "SQL joins ke bare me docs dhundho", "expected": {"action": "call", "name": "external.search_docs", "arguments": {"query": "SQL joins"}}, "output": "{\"action\":\"call\",\"name\":\"external.search_docs\",\"arguments\":{\"query\":\"SQL joins\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "manual_f73e21e0eb10d6617995_hinglish", "language": "hinglish", "backend": "supabase", "operation": "read_file", "question": "/config/app.json file padh kar dikhao", "expected": {"action": "call", "name": "external.read_file", "arguments": {"path": "/config/app.json"}}, "output": "{\"action\":\"call\",\"name\":\"external.read_file\",\"arguments\":{\"path\":\"/config/app.json\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "manual_8a696e3c3522dd69d5ac_en", "language": "en", "backend": "supabase", "operation": "ticket", "question": "Fetch the details for ticket TKT-4486.", "expected": {"action": "call", "name": "external.get_ticket", "arguments": {"ticket_id": "TKT-4486"}}, "output": "{\"action\":\"call\",\"name\":\"external.get_ticket\",\"arguments\":{\"ticket_id\":\"TKT-4486\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "manual_f73e21e0eb10d6617995_en", "language": "en", "backend": "supabase", "operation": "read_file", "question": "Open /config/app.json and read its text.", "expected": {"action": "call", "name": "external.read_file", "arguments": {"path": "/config/app.json"}}, "output": "{\"action\":\"call\",\"name\":\"external.read_file\",\"arguments\":{\"path\":\"/config/app.json\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "manual_8a696e3c3522dd69d5ac_hi", "language": "hi", "backend": "supabase", "operation": "ticket", "question": "टिकट TKT-4486 का विवरण लाओ।", "expected": {"action": "call", "name": "external.get_ticket", "arguments": {"ticket_id": "TKT-4486"}}, "output": "{\"action\":\"call\",\"name\":\"external.get_ticket\",\"arguments\":{\"ticket_id\":\"TKT-4486\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "manual_f73e21e0eb10d6617995_hi", "language": "hi", "backend": "supabase", "operation": "read_file", "question": "/config/app.json फ़ाइल पढ़कर दिखाओ।", "expected": {"action": "call", "name": "external.read_file", "arguments": {"path": "/config/app.json"}}, "output": "{\"action\":\"call\",\"name\":\"external.read_file\",\"arguments\":{\"path\":\"/config/app.json\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "manual_252c6053295d69a93c94_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "search", "question": "docs find database backups", "expected": {"action": "call", "name": "external.search_docs", "arguments": {"query": "database backups"}}, "output": "{\"action\":\"call\",\"name\":\"external.search_docs\",\"arguments\":{\"query\":\"database backups\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "manual_5ebc52b3d20bde2076b3_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "ticket", "question": "ticket TKT-6124 details", "expected": {"action": "call", "name": "external.get_ticket", "arguments": {"ticket_id": "TKT-6124"}}, "output": "{\"action\":\"call\",\"name\":\"external.get_ticket\",\"arguments\":{\"ticket_id\":\"TKT-6124\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "manual_754691bd1eea25d41ccf_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "weather", "question": "Delhi weather fahrenheit pls", "expected": {"action": "call", "name": "external.get_weather", "arguments": {"city": "Delhi", "unit": "fahrenheit"}}, "output": "{\"action\":\"call\",\"name\":\"external.get_weather\",\"arguments\":{\"city\":\"Delhi\",\"unit\":\"fahrenheit\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "manual_252c6053295d69a93c94_en", "language": "en", "backend": "mysql", "operation": "search", "question": "Find documentation about database backups.", "expected": {"action": "call", "name": "external.search_docs", "arguments": {"query": "database backups"}}, "output": "{\"action\":\"call\",\"name\":\"external.search_docs\",\"arguments\":{\"query\":\"database backups\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "manual_4e42bbd40467b2b8c27f_hi", "language": "hi", "backend": "supabase", "operation": "search", "question": "SQL joins के बारे में दस्तावेज़ खोजो।", "expected": {"action": "call", "name": "external.search_docs", "arguments": {"query": "SQL joins"}}, "output": "{\"action\":\"call\",\"name\":\"external.search_docs\",\"arguments\":{\"query\":\"SQL joins\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "manual_5ebc52b3d20bde2076b3_hinglish", "language": "hinglish", "backend": "mysql", "operation": "ticket", "question": "ticket TKT-6124 ki details lao", "expected": {"action": "call", "name": "external.get_ticket", "arguments": {"ticket_id": "TKT-6124"}}, "output": "{\"action\":\"call\",\"name\":\"external.get_ticket\",\"arguments\":{\"ticket_id\":\"TKT-6124\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "manual_754691bd1eea25d41ccf_hinglish", "language": "hinglish", "backend": "mysql", "operation": "weather", "question": "Delhi ka weather fahrenheit me batao", "expected": {"action": "call", "name": "external.get_weather", "arguments": {"city": "Delhi", "unit": "fahrenheit"}}, "output": "{\"action\":\"call\",\"name\":\"external.get_weather\",\"arguments\":{\"city\":\"Delhi\",\"unit\":\"fahrenheit\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "manual_252c6053295d69a93c94_hinglish", "language": "hinglish", "backend": "mysql", "operation": "search", "question": "database backups ke bare me docs dhundho", "expected": {"action": "call", "name": "external.search_docs", "arguments": {"query": "database backups"}}, "output": "{\"action\":\"call\",\"name\":\"external.search_docs\",\"arguments\":{\"query\":\"database backups\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "manual_6f3055c580b33340c854_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "read_file", "question": "/notes/setup.txt read pls", "expected": {"action": "call", "name": "external.read_file", "arguments": {"path": "/notes/setup.txt"}}, "output": "{\"action\":\"call\",\"name\":\"external.read_file\",\"arguments\":{\"path\":\"/notes/setup.txt\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "manual_5ebc52b3d20bde2076b3_en", "language": "en", "backend": "mysql", "operation": "ticket", "question": "Fetch the details for ticket TKT-6124.", "expected": {"action": "call", "name": "external.get_ticket", "arguments": {"ticket_id": "TKT-6124"}}, "output": "{\"action\":\"call\",\"name\":\"external.get_ticket\",\"arguments\":{\"ticket_id\":\"TKT-6124\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "manual_6f3055c580b33340c854_hinglish", "language": "hinglish", "backend": "mysql", "operation": "read_file", "question": "/notes/setup.txt file padh kar dikhao", "expected": {"action": "call", "name": "external.read_file", "arguments": {"path": "/notes/setup.txt"}}, "output": "{\"action\":\"call\",\"name\":\"external.read_file\",\"arguments\":{\"path\":\"/notes/setup.txt\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "manual_6f3055c580b33340c854_en", "language": "en", "backend": "mysql", "operation": "read_file", "question": "Open /notes/setup.txt and read its text.", "expected": {"action": "call", "name": "external.read_file", "arguments": {"path": "/notes/setup.txt"}}, "output": "{\"action\":\"call\",\"name\":\"external.read_file\",\"arguments\":{\"path\":\"/notes/setup.txt\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "manual_754691bd1eea25d41ccf_en", "language": "en", "backend": "mysql", "operation": "weather", "question": "What is the weather like in Delhi? Use fahrenheit.", "expected": {"action": "call", "name": "external.get_weather", "arguments": {"city": "Delhi", "unit": "fahrenheit"}}, "output": "{\"action\":\"call\",\"name\":\"external.get_weather\",\"arguments\":{\"city\":\"Delhi\",\"unit\":\"fahrenheit\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "manual_754691bd1eea25d41ccf_hi", "language": "hi", "backend": "mysql", "operation": "weather", "question": "Delhi का मौसम fahrenheit में बताओ।", "expected": {"action": "call", "name": "external.get_weather", "arguments": {"city": "Delhi", "unit": "fahrenheit"}}, "output": "{\"action\":\"call\",\"name\":\"external.get_weather\",\"arguments\":{\"city\":\"Delhi\",\"unit\":\"fahrenheit\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "manual_5ebc52b3d20bde2076b3_hi", "language": "hi", "backend": "mysql", "operation": "ticket", "question": "टिकट TKT-6124 का विवरण लाओ।", "expected": {"action": "call", "name": "external.get_ticket", "arguments": {"ticket_id": "TKT-6124"}}, "output": "{\"action\":\"call\",\"name\":\"external.get_ticket\",\"arguments\":{\"ticket_id\":\"TKT-6124\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "manual_6f3055c580b33340c854_hi", "language": "hi", "backend": "mysql", "operation": "read_file", "question": "/notes/setup.txt फ़ाइल पढ़कर दिखाओ।", "expected": {"action": "call", "name": "external.read_file", "arguments": {"path": "/notes/setup.txt"}}, "output": "{\"action\":\"call\",\"name\":\"external.read_file\",\"arguments\":{\"path\":\"/notes/setup.txt\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "manual_252c6053295d69a93c94_hi", "language": "hi", "backend": "mysql", "operation": "search", "question": "database backups के बारे में दस्तावेज़ खोजो।", "expected": {"action": "call", "name": "external.search_docs", "arguments": {"query": "database backups"}}, "output": "{\"action\":\"call\",\"name\":\"external.search_docs\",\"arguments\":{\"query\":\"database backups\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}