[
  {
    "model": "phi3-BASE-Q4_K_M.gguf",
    "n": 12,
    "execution_accuracy_pct": 41.7,
    "valid_sql_pct": 100.0,
    "exact_match_pct": 8.3,
    "avg_latency_s": 7.12,
    "avg_decode_tok_s": 5.5,
    "details": [
      {
        "q": "List all employees in department 1.",
        "pred": "SELECT employees.name FROM employees WHERE employees.department_id = 1;",
        "gold": "SELECT * FROM employees WHERE department_id = 1;",
        "valid": true,
        "exec_match": false,
        "gold_err": null
      },
      {
        "q": "Which software products cost less than 500 dollars?",
        "pred": "SELECT DISTINCT products.name\nFROM products\nJOIN sales ON products.id = sales.product_id\nWHERE products.category = 'Software' AND products.price < 500;",
        "gold": "SELECT * FROM products WHERE category = 'Software' AND price < 500;",
        "valid": true,
        "exec_match": false,
        "gold_err": null
      },
      {
        "q": "How many departments do not have a manager?",
        "pred": "SELECT COUNT(*) FROM departments WHERE manager_id IS NULL;",
        "gold": "SELECT COUNT(*) FROM departments WHERE manager_id IS NULL;",
        "valid": true,
        "exec_match": true,
        "gold_err": null
      },
      {
        "q": "Find the total salary budget for the Engineering department.",
        "pred": "SELECT SUM(e.salary) AS total_salary_budget\nFROM employees e\nJOIN departments d ON e.department_id = d.id\nWHERE d.name = 'Engineering';",
        "gold": "SELECT SUM(e.salary) FROM employees e JOIN departments d ON e.department_id = d.id WHERE d.name = 'Engineering';",
        "valid": true,
        "exec_match": true,
        "gold_err": null
      },
      {
        "q": "What is the total quantity of CRM products sold in 2024?",
        "pred": "SELECT SUM(sales.quantity) AS total_quantity\nFROM sales\nJOIN products ON sales.product_id = products.id\nWHERE products.category = 'Software' AND strftime('%Y', sales.sale_date) = '2024';",
        "gold": "SELECT SUM(s.quantity) FROM sales s JOIN products p ON s.product_id = p.id WHERE p.name = 'Enterprise CRM' AND s.sale_date BETWEEN '2024-01-01' AND '2024-12-31';",
        "valid": true,
        "exec_match": false,
        "gold_err": null
      },
      {
        "q": "Show the employee with the highest salary in department 2.",
        "pred": "SELECT employees.name\nFROM employees\nJOIN departments ON employees.department_id = departments.id\nWHERE departments.id = 2\nORDER BY employees.salary DESC\nLIMIT 1;",
        "gold": "SELECT name, salary FROM employees WHERE department_id = 2 ORDER BY salary DESC LIMIT 1;",
        "valid": true,
        "exec_match": false,
        "gold_err": null
      },
      {
        "q": "List employees who report directly to Alice Vance.",
        "pred": "SELECT e.id, e.name\nFROM employees e\nJOIN employees m ON e.manager_id = m.id\nWHERE m.name = 'Alice Vance';",
        "gold": "SELECT e.name FROM employees e JOIN employees m ON e.manager_id = m.id WHERE m.name = 'Alice Vance';",
        "valid": true,
        "exec_match": false,
        "gold_err": null
      },
      {
        "q": "Find the average sale amount for employee ID 9.",
        "pred": "SELECT AVG(sales.amount) AS average_sale_amount\nFROM sales\nWHERE sales.employee_id = 9;",
        "gold": "SELECT AVG(amount) FROM sales WHERE employee_id = 9;",
        "valid": true,
        "exec_match": true,
        "gold_err": null
      },
      {
        "q": "Find the total revenue generated from software category products.",
        "pred": "SELECT SUM(sales.amount * sales.quantity) AS total_revenue\nFROM sales\nJOIN products ON sales.product_id = products.id\nWHERE products.category = 'Software';",
        "gold": "SELECT SUM(s.amount) FROM sales s JOIN products p ON s.product_id = p.id WHERE p.category = 'Software';",
        "valid": true,
        "exec_match": false,
        "gold_err": null
      },
      {
        "q": "Which product has the lowest price?",
        "pred": "SELECT products.name, products.price\nFROM products\nORDER BY products.price ASC\nLIMIT 1;",
        "gold": "SELECT name, price FROM products ORDER BY price ASC LIMIT 1;",
        "valid": true,
        "exec_match": true,
        "gold_err": null
      },
      {
        "q": "Find the employee who generated the highest total revenue across all sales.",
        "pred": "SELECT employees.name, SUM(sales.amount * sales.quantity) AS total_revenue\nFROM employees\nJOIN sales ON employees.id = sales.employee_id\nGROUP BY employees.id\nORDER BY total_revenue DESC\nLIMIT 1;",
        "gold": "SELECT e.name, SUM(s.amount) AS total_revenue FROM employees e JOIN sales s ON e.id = s.employee_id GROUP BY e.name ORDER BY total_revenue DESC LIMIT 1;",
        "valid": true,
        "exec_match": false,
        "gold_err": null
      },
      {
        "q": "List products sold in November 2024.",
        "pred": "SELECT DISTINCT products.name\nFROM products\nJOIN sales ON products.id = sales.product_id\nWHERE strftime('%Y-%m', sale_date) = '2024-11';",
        "gold": "SELECT DISTINCT p.name FROM products p JOIN sales s ON p.id = s.product_id WHERE s.sale_date BETWEEN '2024-11-01' AND '2024-11-30';",
        "valid": true,
        "exec_match": true,
        "gold_err": null
      }
    ]
  }
]