Spaces:
Sleeping
Sleeping
File size: 5,344 Bytes
1d8be28 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 | [
{
"model": "phi3-BASE-Q4_K_M.gguf",
"n": 12,
"execution_accuracy_pct": 41.7,
"valid_sql_pct": 100.0,
"exact_match_pct": 8.3,
"avg_latency_s": 7.12,
"avg_decode_tok_s": 5.5,
"details": [
{
"q": "List all employees in department 1.",
"pred": "SELECT employees.name FROM employees WHERE employees.department_id = 1;",
"gold": "SELECT * FROM employees WHERE department_id = 1;",
"valid": true,
"exec_match": false,
"gold_err": null
},
{
"q": "Which software products cost less than 500 dollars?",
"pred": "SELECT DISTINCT products.name\nFROM products\nJOIN sales ON products.id = sales.product_id\nWHERE products.category = 'Software' AND products.price < 500;",
"gold": "SELECT * FROM products WHERE category = 'Software' AND price < 500;",
"valid": true,
"exec_match": false,
"gold_err": null
},
{
"q": "How many departments do not have a manager?",
"pred": "SELECT COUNT(*) FROM departments WHERE manager_id IS NULL;",
"gold": "SELECT COUNT(*) FROM departments WHERE manager_id IS NULL;",
"valid": true,
"exec_match": true,
"gold_err": null
},
{
"q": "Find the total salary budget for the Engineering department.",
"pred": "SELECT SUM(e.salary) AS total_salary_budget\nFROM employees e\nJOIN departments d ON e.department_id = d.id\nWHERE d.name = 'Engineering';",
"gold": "SELECT SUM(e.salary) FROM employees e JOIN departments d ON e.department_id = d.id WHERE d.name = 'Engineering';",
"valid": true,
"exec_match": true,
"gold_err": null
},
{
"q": "What is the total quantity of CRM products sold in 2024?",
"pred": "SELECT SUM(sales.quantity) AS total_quantity\nFROM sales\nJOIN products ON sales.product_id = products.id\nWHERE products.category = 'Software' AND strftime('%Y', sales.sale_date) = '2024';",
"gold": "SELECT SUM(s.quantity) FROM sales s JOIN products p ON s.product_id = p.id WHERE p.name = 'Enterprise CRM' AND s.sale_date BETWEEN '2024-01-01' AND '2024-12-31';",
"valid": true,
"exec_match": false,
"gold_err": null
},
{
"q": "Show the employee with the highest salary in department 2.",
"pred": "SELECT employees.name\nFROM employees\nJOIN departments ON employees.department_id = departments.id\nWHERE departments.id = 2\nORDER BY employees.salary DESC\nLIMIT 1;",
"gold": "SELECT name, salary FROM employees WHERE department_id = 2 ORDER BY salary DESC LIMIT 1;",
"valid": true,
"exec_match": false,
"gold_err": null
},
{
"q": "List employees who report directly to Alice Vance.",
"pred": "SELECT e.id, e.name\nFROM employees e\nJOIN employees m ON e.manager_id = m.id\nWHERE m.name = 'Alice Vance';",
"gold": "SELECT e.name FROM employees e JOIN employees m ON e.manager_id = m.id WHERE m.name = 'Alice Vance';",
"valid": true,
"exec_match": false,
"gold_err": null
},
{
"q": "Find the average sale amount for employee ID 9.",
"pred": "SELECT AVG(sales.amount) AS average_sale_amount\nFROM sales\nWHERE sales.employee_id = 9;",
"gold": "SELECT AVG(amount) FROM sales WHERE employee_id = 9;",
"valid": true,
"exec_match": true,
"gold_err": null
},
{
"q": "Find the total revenue generated from software category products.",
"pred": "SELECT SUM(sales.amount * sales.quantity) AS total_revenue\nFROM sales\nJOIN products ON sales.product_id = products.id\nWHERE products.category = 'Software';",
"gold": "SELECT SUM(s.amount) FROM sales s JOIN products p ON s.product_id = p.id WHERE p.category = 'Software';",
"valid": true,
"exec_match": false,
"gold_err": null
},
{
"q": "Which product has the lowest price?",
"pred": "SELECT products.name, products.price\nFROM products\nORDER BY products.price ASC\nLIMIT 1;",
"gold": "SELECT name, price FROM products ORDER BY price ASC LIMIT 1;",
"valid": true,
"exec_match": true,
"gold_err": null
},
{
"q": "Find the employee who generated the highest total revenue across all sales.",
"pred": "SELECT employees.name, SUM(sales.amount * sales.quantity) AS total_revenue\nFROM employees\nJOIN sales ON employees.id = sales.employee_id\nGROUP BY employees.id\nORDER BY total_revenue DESC\nLIMIT 1;",
"gold": "SELECT e.name, SUM(s.amount) AS total_revenue FROM employees e JOIN sales s ON e.id = s.employee_id GROUP BY e.name ORDER BY total_revenue DESC LIMIT 1;",
"valid": true,
"exec_match": false,
"gold_err": null
},
{
"q": "List products sold in November 2024.",
"pred": "SELECT DISTINCT products.name\nFROM products\nJOIN sales ON products.id = sales.product_id\nWHERE strftime('%Y-%m', sale_date) = '2024-11';",
"gold": "SELECT DISTINCT p.name FROM products p JOIN sales s ON p.id = s.product_id WHERE s.sale_date BETWEEN '2024-11-01' AND '2024-11-30';",
"valid": true,
"exec_match": true,
"gold_err": null
}
]
}
] |