Update evaluation_mode.py & interactive_mode.py version 1.1.0

- Update scripts to automate the testing and evaluation.
- Write into file for comparison.

backend/Q&A_report_Qwen_Qwen2.5-Coder-32B-Instruct_featherless-ai.csv

@@ -0,0 +1,29 @@
+id,model_id,question,status,data_returned_correct,attempts,nl_response,sql_generated,error
+1,Qwen/Qwen2.5-Coder-32B-Instruct:featherless-ai,List all the artists name in the database.,success,True,1,"The database contains a wide range of artists, including AC/DC, Accept, Aerosmith, Alanis Morissette, Alice In Chains, and many others.",SELECT Name FROM Artist,
+2,Qwen/Qwen2.5-Coder-32B-Instruct:featherless-ai,How many genres are there?,success,True,1,There are 25 genres.,SELECT COUNT(*) FROM Genre,
+3,Qwen/Qwen2.5-Coder-32B-Instruct:featherless-ai,List the names of the first 5 tracks.,success,True,1,"The first 5 tracks are ""For Those About To Rock (We Salute You)"", ""Balls to the Wall"", ""Fast As a Shark"", ""Restless and Wild"", and ""Princess of the Dawn"".",SELECT Name FROM Track LIMIT 5,
+4,Qwen/Qwen2.5-Coder-32B-Instruct:featherless-ai,Count the number of customers located in the USA.,success,True,1,There are 13 customers located in the USA.,SELECT COUNT(*) FROM Customer WHERE Country = 'USA',
+5,Qwen/Qwen2.5-Coder-32B-Instruct:featherless-ai,Find all invoices for the customer with ID 1.,success,True,1,There are 7 invoices for the customer with ID 1.,SELECT * FROM Invoice WHERE CustomerId = 1,
+6,Qwen/Qwen2.5-Coder-32B-Instruct:featherless-ai,List each album title along with the artist's name.,success,True,1,The database contains a list of over 50 albums along with their respective artists.,"SELECT Album.Title, Artist.Name FROM Album JOIN Artist ON Album.ArtistId = Artist.ArtistId",
+7,Qwen/Qwen2.5-Coder-32B-Instruct:featherless-ai,How many tracks belong to the 'Rock' genre?,success,True,1,There are 1297 tracks that belong to the 'Rock' genre.,SELECT COUNT(*) FROM Track AS T JOIN Genre AS G ON T.GenreId = G.GenreId WHERE G.Name = 'Rock',
+8,Qwen/Qwen2.5-Coder-32B-Instruct:featherless-ai,Show the total revenue generated from each country.,success,True,1,"The total revenue generated spans multiple countries, with the USA leading at $523.06, followed by Canada at $303.96, France at $195.10, Brazil at $190.10, and Portugal at $77.24.","SELECT BillingCountry, SUM(Total) AS TotalRevenue FROM Invoice GROUP BY BillingCountry",
+9,Qwen/Qwen2.5-Coder-32B-Instruct:featherless-ai,Find the total number of items sold for each media type.,success,True,1,"The total number of items sold includes 1976 MPEG audio files, 146 protected AAC audio files, 111 protected MPEG-4 video files, and 7 AAC audio files (3 regular and 4 purchased).","SELECT MediaType.Name, COUNT(*) AS TotalItemsSold
+FROM MediaType
+JOIN Track ON MediaType.MediaTypeId = Track.MediaTypeId
+JOIN InvoiceLine ON Track.TrackId = InvoiceLine.TrackId
+GROUP BY MediaType.Name",
+10,Qwen/Qwen2.5-Coder-32B-Instruct:featherless-ai,List the first and last names of all employees who are Sales Support Agents.,success,True,1,"The Sales Support Agents are Jane Peacock, Margaret Park, and Steve Johnson.","SELECT FirstName, LastName FROM Employee WHERE Title = 'Sales Support Agent'",
+11,Qwen/Qwen2.5-Coder-32B-Instruct:featherless-ai,List the top 5 customers who have spent the most money in total.,success,False,1,"The top 5 customers who have spent the most money in total are Helena Holý with $49.62, Richard Cunningham with $47.62, Luis Rojas with $46.62, Ladislav Kovács with $45.62, and Hugh O'Reilly with $45.62.","SELECT c.CustomerId, c.FirstName, c.LastName, SUM(i.Total) AS TotalSpent
+FROM Customer c
+JOIN Invoice i ON c.CustomerId = i.CustomerId
+GROUP BY c.CustomerId, c.FirstName, c.LastName
+ORDER BY TotalSpent DESC
+LIMIT 5",
+12,Qwen/Qwen2.5-Coder-32B-Instruct:featherless-ai,Which artist has the most tracks in the database? Give the name and count.,success,True,1,Iron Maiden has the most tracks in the database with 213 tracks.,"SELECT T2.Name, COUNT(T3.TrackId) AS TrackCount FROM Album AS T1 JOIN Artist AS T2 ON T1.ArtistId = T2.ArtistId JOIN Track AS T3 ON T1.AlbumId = T3.AlbumId GROUP BY T2.Name ORDER BY TrackCount DESC LIMIT 1",
+13,Qwen/Qwen2.5-Coder-32B-Instruct:featherless-ai,Which genres have more than 100 tracks? List the genre name and count.,success,True,1,"Rock has 1297 tracks, Jazz has 130 tracks, Metal has 374 tracks, Alternative & Punk has 332 tracks, and Latin has 579 tracks.","SELECT G.Name, COUNT(T.TrackId) AS TrackCount
+FROM Genre G
+JOIN Track T ON G.GenreId = T.GenreId
+GROUP BY G.GenreId
+HAVING COUNT(T.TrackId) > 100",
+14,Qwen/Qwen2.5-Coder-32B-Instruct:featherless-ai,Calculate the average track length in seconds for each genre.,success,False,1,"The average track lengths vary significantly across genres, ranging from approximately 134.64 seconds to 2625.55 seconds.","SELECT GenreId, AVG(Milliseconds / 1000.0) AS AverageLengthInSeconds FROM Track GROUP BY GenreId",
+15,Qwen/Qwen2.5-Coder-32B-Instruct:featherless-ai,Identify the artist who has earned the most revenue from customers in Canada.,success,False,2,Metallica has earned the most revenue from customers in Canada.,SELECT T3.Name FROM Invoice AS T1 JOIN Customer AS T2 ON T1.CustomerId = T2.CustomerId JOIN InvoiceLine AS T4 ON T1.InvoiceId = T4.InvoiceId JOIN Track AS T5 ON T4.TrackId = T5.TrackId JOIN Album AS T6 ON T5.AlbumId = T6.AlbumId JOIN Artist AS T3 ON T6.ArtistId = T3.ArtistId WHERE T2.Country = 'Canada' GROUP BY T3.Name ORDER BY SUM(T1.Total) DESC LIMIT 1,

backend/Q&A_report_Qwen_Qwen2.5-Coder-7B-Instruct_featherless-ai.csv

@@ -0,0 +1,46 @@
+id,model_id,question,status,data_returned_correct,attempts,nl_response,sql_generated,error
+1,Qwen/Qwen2.5-Coder-7B-Instruct:featherless-ai,List all the artists name in the database.,success,True,1,There are 163 artists listed in the database.,SELECT Name FROM Artist,
+2,Qwen/Qwen2.5-Coder-7B-Instruct:featherless-ai,How many genres are there?,success,True,1,There are 25 genres.,SELECT COUNT(*) FROM Genre,
+3,Qwen/Qwen2.5-Coder-7B-Instruct:featherless-ai,List the names of the first 5 tracks.,success,True,1,"The first 5 tracks are: For Those About To Rock (We Salute You), Balls to the Wall, Fast As a Shark, Restless and Wild, and Princess of the Dawn.",SELECT Name FROM Track LIMIT 5,
+4,Qwen/Qwen2.5-Coder-7B-Instruct:featherless-ai,Count the number of customers located in the USA.,success,True,1,There are 13 customers located in the USA.,SELECT COUNT(*) FROM Customer WHERE Country = 'USA';,
+5,Qwen/Qwen2.5-Coder-7B-Instruct:featherless-ai,Find all invoices for the customer with ID 1.,success,True,1,The customer with ID 1 has 7 invoices.,SELECT * FROM Invoice WHERE CustomerId = 1;,
+6,Qwen/Qwen2.5-Coder-7B-Instruct:featherless-ai,List each album title along with the artist's name.,success,True,1,"There are 100 albums listed, each with its corresponding artist's name.","SELECT T1.Title, T2.Name FROM Album AS T1 INNER JOIN Artist AS T2 ON T1.ArtistId = T2.ArtistId",
+7,Qwen/Qwen2.5-Coder-7B-Instruct:featherless-ai,How many tracks belong to the 'Rock' genre?,success,True,1,There are 1297 tracks that belong to the 'Rock' genre.,SELECT COUNT(*) FROM Track JOIN Genre ON Track.GenreId = Genre.GenreId WHERE Genre.Name = 'Rock',
+8,Qwen/Qwen2.5-Coder-7B-Instruct:featherless-ai,Show the total revenue generated from each country.,success,True,1,"Total revenue generated from each country:
+
+- Argentina: $37.62
+- Australia: $37.62
+- Austria: $42.62
+- Belgium: $37.62
+- Brazil: $190.1
+- Canada: $303.96
+- Chile: $46.62
+- Czech Republic: $90.24
+- Denmark: $37.62
+- Finland: $41.62
+- France: $195.1
+- Germany: $156.48
+- Hungary: $45.62
+- India: $75.26
+- Ireland: $45.62
+- Italy: $37.62
+- Netherlands: $40.62
+- Norway: $39.62
+- Poland: $37.62
+- Portugal: $77.24
+- Spain: $37.62
+- Sweden: $38.62
+- USA: $523.06
+- United Kingdom: $112.86","SELECT BillingCountry, SUM(Total) AS TotalRevenue FROM Invoice GROUP BY BillingCountry",
+9,Qwen/Qwen2.5-Coder-7B-Instruct:featherless-ai,Find the total number of items sold for each media type.,Error executing SQL after 3 attempts,False,0,N/A,"SELECT T1.Name, COUNT(T2.MediaTypeId) AS TotalItemsSold FROM MediaType AS T1 INNER JOIN Invoices AS T2 ON T1.MediaTypeId = T2.MediaTypeId GROUP BY T1.Name",no such table: Invoices
+10,Qwen/Qwen2.5-Coder-7B-Instruct:featherless-ai,List the first and last names of all employees who are Sales Support Agents.,success,True,1,"The first and last names of all Sales Support Agents are Jane Peacock, Margaret Park, and Steve Johnson.","SELECT FirstName, LastName FROM Employee WHERE Title = 'Sales Support Agent'",
+11,Qwen/Qwen2.5-Coder-7B-Instruct:featherless-ai,List the top 5 customers who have spent the most money in total.,success,False,2,"The top 5 customers who have spent the most money in total are Helena Holý, Richard Cunningham, Luis Rojas, Ladislav Kovács, and Hugh O'Reilly, each with a total expenditure of $49.62.","SELECT c.CustomerId, c.FirstName, c.LastName, SUM(i.Total) AS TotalSpent
+FROM Customer c
+JOIN Invoice i ON c.CustomerId = i.CustomerId
+GROUP BY c.CustomerId
+ORDER BY TotalSpent DESC
+LIMIT 5",
+12,Qwen/Qwen2.5-Coder-7B-Instruct:featherless-ai,Which artist has the most tracks in the database? Give the name and count.,success,True,3,Iron Maiden has the most tracks with 213.,"SELECT T3.Name, COUNT(*) as TrackCount FROM Track AS T1 INNER JOIN Album AS T2 ON T1.AlbumId = T2.AlbumId INNER JOIN Artist AS T3 ON T2.ArtistId = T3.ArtistId GROUP BY T3.Name ORDER BY TrackCount DESC LIMIT 1",
+13,Qwen/Qwen2.5-Coder-7B-Instruct:featherless-ai,Which genres have more than 100 tracks? List the genre name and count.,success,True,1,"Genres with more than 100 tracks include Rock (1297), Metal (374), Latin (579), Jazz (130), and Alternative & Punk (332).","SELECT T2.Name, COUNT(T1.TrackId) AS TrackCount FROM TRACK AS T1 INNER JOIN GENRE AS T2 ON T1.GenreId = T2.GenreId GROUP BY T2.Name HAVING COUNT(T1.TrackId) > 100",
+14,Qwen/Qwen2.5-Coder-7B-Instruct:featherless-ai,Calculate the average track length in seconds for each genre.,success,False,1,The average track length in seconds for each genre ranges from 134.17 seconds (genre 5) to 2934.05 seconds (genre 24).,"SELECT AVG(Milliseconds / 1000) AS AverageLengthInSeconds, GenreId FROM Track GROUP BY GenreId",
+15,Qwen/Qwen2.5-Coder-7B-Instruct:featherless-ai,Identify the artist who has earned the most revenue from customers in Canada.,Error executing SQL after 3 attempts,False,0,N/A,SELECT T1.Name FROM ARTIST AS T1 JOIN ALBUM AS T2 ON T1.ArtistId = T2.ArtistId JOIN TRACK AS T3 ON T2.AlbumId = T3.AlbumId JOIN INVOICEITEM AS T4 ON T3.TrackId = T4.TrackId JOIN CUSTOMER AS T5 ON T4.InvoiceId = T5.InvoiceId WHERE T5.Country = 'Canada' GROUP BY T1.Name ORDER BY SUM(T4.UnitPrice * T4.Quantity) DESC LIMIT 1,no such table: INVOICEITEM

backend/Q&A_report_defog_sqlcoder-7b-2.csv

@@ -0,0 +1,21 @@
+id,model_id,question,status,data_returned_correct,attempts,nl_response,sql_generated,error
+1,defog/sqlcoder-7b-2,List all the artists name in the database.,success,True,1,,"SELECT a.""Name"" FROM ""Artist"" a;",
+2,defog/sqlcoder-7b-2,How many genres are there?,success,True,1, There are 25 genres in the database.,SELECT COUNT(g.genreid) AS number_of_genres FROM genre g;,
+3,defog/sqlcoder-7b-2,List the names of the first 5 tracks.,success,True,1," The first 5 tracks are:
+
+ [('For Those About To Rock (We Salute You)',), ('Balls to the Wall',), ('Fast As a Shark',), ('Restless and Wild',), ('Princess of the Dawn',)]","SELECT t.name FROM ""Track"" t LIMIT 5;",
+4,defog/sqlcoder-7b-2,Count the number of customers located in the USA.,Error executing SQL after 3 attempts,False,0,N/A,"SELECT COUNT(*) FROM ""Customer"" c WHERE c.Country ILIKE '%USA%';","near ""ILIKE"": syntax error"
+5,defog/sqlcoder-7b-2,Find all invoices for the customer with ID 1.,success,True,1," The user asked the following question: ""Find all invoices for the customer with ID 1.""",SELECT * FROM Invoice i WHERE i.CustomerId = 1;,
+6,defog/sqlcoder-7b-2,List each album title along with the artist's name.,Error executing SQL after 3 attempts,True,0,N/A,"SELECT a.""Title"", ar.""Name"" AS ""Artist Name"" FROM ""Album"" a JOIN ""Artist"" ar ON a.""ArtistId"" = ar.""ArtistId"";","(Request ID: Root=1-6a08e19d-1afb35d014009a893606e2f5;074f3f39-9bf6-45d9-a1be-4a596301074a)
+
+Bad request:
+{'message': ""This model's maximum context length is 4096 tokens. However, you requested 512 output tokens and your prompt contains at least 3585 input tokens, for a total of at least 4097 tokens. Please reduce the length of the input prompt or the number of requested output tokens. (parameter=input_tokens, value=3585)"", 'type': 'invalid_request_error', 'param': None, 'code': 'bad_request'}"
+7,defog/sqlcoder-7b-2,How many tracks belong to the 'Rock' genre?,success,True,1, There are 1297 tracks in the 'Rock' genre.,"SELECT COUNT(t.TrackId) AS number_of_tracks FROM ""Track"" t JOIN ""Genre"" g ON t.GenreId = g.GenreId WHERE g.Name = 'Rock';",
+8,defog/sqlcoder-7b-2,Show the total revenue generated from each country.,Error executing SQL after 3 attempts,False,0,N/A,"SELECT c.country_name, SUM(i.total) AS total_revenue FROM ""Invoice"" i JOIN ""Customer"" c ON i.customer_id = c.customer_id GROUP BY c.country_name ORDER BY total_revenue DESC NULLS LAST;",no such column: c.country_name
+9,defog/sqlcoder-7b-2,Find the total number of items sold for each media type.,Error executing SQL after 3 attempts,False,0,N/A,"SELECT m.name AS media_type, COUNT(*) AS total_items_sold FROM ""MediaType"" m JOIN ""Sales"" s ON m.media_type_id = s.media_type_id GROUP BY m.name;",no such table: Sales
+10,defog/sqlcoder-7b-2,List the first and last names of all employees who are Sales Support Agents.,Error executing SQL after 3 attempts,False,0,N/A,"SELECT e.first_name, e.last_name FROM ""Employee"" e WHERE e.title = 'Sales Support Agent';",no such column: e.first_name
+11,defog/sqlcoder-7b-2,List the top 5 customers who have spent the most money in total.,Error executing SQL after 3 attempts,False,0,N/A,"SELECT c.customerid, c.firstname, c.lastname, SUM(oi.productid * oi.quantity) AS total_spent FROM customer c JOIN orders o ON c.customerid = o.customerid JOIN orderitems oi ON o.orderid = oi.orderid GROUP BY c.customerid, c.firstname, c.lastname ORDER BY total_spent DESC LIMIT 5;",no such table: orders
+12,defog/sqlcoder-7b-2,Which artist has the most tracks in the database? Give the name and count.,success,False,2," The artist with the most tracks in the database is ""The Beatles"" with 52 tracks.",,
+13,defog/sqlcoder-7b-2,Which genres have more than 100 tracks? List the genre name and count.,success,True,1," The database returned more than 5 rows, so I will provide a brief summary sentence.","SELECT g.name, COUNT(t.trackid) AS COUNT FROM genre g JOIN track t ON g.genreid = t.genreid GROUP BY g.name HAVING COUNT(t.trackid) > 100 ORDER BY COUNT DESC NULLS LAST;",
+14,defog/sqlcoder-7b-2,Calculate the average track length in seconds for each genre.,Error executing SQL after 3 attempts,False,0,N/A,"SELECT g.Name, AVG(t.Milliseconds / 1000)) AS AverageDuration FROM Track t JOIN Genre g ON t.GenreId = g.GenreId GROUP BY g.Name ORDER BY AverageDuration DESC NULLS LAST;","near "")"": syntax error"
+15,defog/sqlcoder-7b-2,Identify the artist who has earned the most revenue from customers in Canada.,Error executing SQL after 3 attempts,False,0,N/A,"SELECT a.""Name"" FROM ""Artist"" a JOIN ""Album"" al ON a.""ArtistId"" = al.""AlbumArtistId"" WHERE al.""AlbumCountry"" = 'Canada' ORDER BY a.""Name"" DESC NULLS LAST LIMIT 1;",no such column: al.AlbumCountry

backend/sql_eval_Qwen_Qwen2.5-Coder-32B-Instruct_featherless-ai.json

@@ -0,0 +1,137 @@
+[
+    {
+        "id": 1,
+        "question": "List all the artists name in the database.",
+        "taxonomy": "Selection",
+        "ex_pass": true,
+        "esm_pass": true,
+        "generated_sql": "SELECT Name FROM Artist",
+        "gold_sql": "SELECT Name FROM Artist;"
+    },
+    {
+        "id": 2,
+        "question": "How many genres are there?",
+        "taxonomy": "Aggregation",
+        "ex_pass": true,
+        "esm_pass": true,
+        "generated_sql": "SELECT COUNT(*) FROM Genre",
+        "gold_sql": "SELECT COUNT(*) FROM Genre;"
+    },
+    {
+        "id": 3,
+        "question": "List the names of the first 5 tracks.",
+        "taxonomy": "Selection, Limit",
+        "ex_pass": true,
+        "esm_pass": true,
+        "generated_sql": "SELECT Name FROM Track LIMIT 5",
+        "gold_sql": "SELECT Name FROM Track LIMIT 5;"
+    },
+    {
+        "id": 4,
+        "question": "Count the number of customers located in the USA.",
+        "taxonomy": "Aggregation, Filtering",
+        "ex_pass": true,
+        "esm_pass": true,
+        "generated_sql": "SELECT COUNT(*) FROM Customer WHERE Country = 'USA'",
+        "gold_sql": "SELECT COUNT(*) FROM Customer WHERE Country = 'USA';"
+    },
+    {
+        "id": 5,
+        "question": "Find all invoices for the customer with ID 1.",
+        "taxonomy": "Selection, Filtering",
+        "ex_pass": true,
+        "esm_pass": true,
+        "generated_sql": "SELECT * FROM Invoice WHERE CustomerId = 1",
+        "gold_sql": "SELECT * FROM Invoice WHERE CustomerId = 1;"
+    },
+    {
+        "id": 6,
+        "question": "List each album title along with the artist's name.",
+        "taxonomy": "Simple Join",
+        "ex_pass": true,
+        "esm_pass": true,
+        "generated_sql": "SELECT Album.Title, Artist.Name FROM Album JOIN Artist ON Album.ArtistId = Artist.ArtistId",
+        "gold_sql": "SELECT Album.Title, Artist.Name FROM Album JOIN Artist ON Album.ArtistId = Artist.ArtistId;"
+    },
+    {
+        "id": 7,
+        "question": "How many tracks belong to the 'Rock' genre?",
+        "taxonomy": "Simple Join, Filtering, Aggregation",
+        "ex_pass": true,
+        "esm_pass": false,
+        "generated_sql": "SELECT COUNT(*) FROM Track AS T JOIN Genre AS G ON T.GenreId = G.GenreId WHERE G.Name = 'Rock'",
+        "gold_sql": "SELECT COUNT(*) FROM Track JOIN Genre ON Track.GenreId = Genre.GenreId WHERE Genre.Name = 'Rock';"
+    },
+    {
+        "id": 8,
+        "question": "Show the total revenue generated from each country.",
+        "taxonomy": "Aggregation, Grouping",
+        "ex_pass": true,
+        "esm_pass": false,
+        "generated_sql": "SELECT BillingCountry, SUM(Total) AS TotalRevenue FROM Invoice GROUP BY BillingCountry",
+        "gold_sql": "SELECT BillingCountry, SUM(Total) FROM Invoice GROUP BY BillingCountry;"
+    },
+    {
+        "id": 9,
+        "question": "Find the total number of items sold for each media type.",
+        "taxonomy": "Multi-Join, Aggregation, Grouping",
+        "ex_pass": true,
+        "esm_pass": false,
+        "generated_sql": "SELECT MediaType.Name, COUNT(*) AS TotalItemsSold\nFROM MediaType\nJOIN Track ON MediaType.MediaTypeId = Track.MediaTypeId\nJOIN InvoiceLine ON Track.TrackId = InvoiceLine.TrackId\nGROUP BY MediaType.Name",
+        "gold_sql": "SELECT MediaType.Name, SUM(InvoiceLine.Quantity) FROM InvoiceLine JOIN Track ON InvoiceLine.TrackId = Track.TrackId JOIN MediaType ON Track.MediaTypeId = MediaType.MediaTypeId GROUP BY MediaType.Name;"
+    },
+    {
+        "id": 10,
+        "question": "List the first and last names of all employees who are Sales Support Agents.",
+        "taxonomy": "Selection, Filtering",
+        "ex_pass": true,
+        "esm_pass": true,
+        "generated_sql": "SELECT FirstName, LastName FROM Employee WHERE Title = 'Sales Support Agent'",
+        "gold_sql": "SELECT FirstName, LastName FROM Employee WHERE Title = 'Sales Support Agent';"
+    },
+    {
+        "id": 11,
+        "question": "List the top 5 customers who have spent the most money in total.",
+        "taxonomy": "Simple Join, Aggregation, Grouping, Ordering, Limit",
+        "ex_pass": false,
+        "esm_pass": false,
+        "generated_sql": "SELECT c.CustomerId, c.FirstName, c.LastName, SUM(i.Total) AS TotalSpent\nFROM Customer c\nJOIN Invoice i ON c.CustomerId = i.CustomerId\nGROUP BY c.CustomerId, c.FirstName, c.LastName\nORDER BY TotalSpent DESC\nLIMIT 5",
+        "gold_sql": "SELECT c.FirstName, c.LastName, SUM(i.Total) as TotalSpent FROM Customer c JOIN Invoice i ON c.CustomerId = i.CustomerId GROUP BY c.CustomerId ORDER BY TotalSpent DESC LIMIT 5;"
+    },
+    {
+        "id": 12,
+        "question": "Which artist has the most tracks in the database? Give the name and count.",
+        "taxonomy": "Multi-Join, Aggregation, Grouping, Ordering, Limit",
+        "ex_pass": true,
+        "esm_pass": false,
+        "generated_sql": "SELECT T2.Name, COUNT(T3.TrackId) AS TrackCount FROM Album AS T1 JOIN Artist AS T2 ON T1.ArtistId = T2.ArtistId JOIN Track AS T3 ON T1.AlbumId = T3.AlbumId GROUP BY T2.Name ORDER BY TrackCount DESC LIMIT 1",
+        "gold_sql": "SELECT ar.Name, COUNT(t.TrackId) as TrackCount FROM Artist ar JOIN Album al ON ar.ArtistId = al.ArtistId JOIN Track t ON al.AlbumId = t.AlbumId GROUP BY ar.ArtistId ORDER BY TrackCount DESC LIMIT 1;"
+    },
+    {
+        "id": 13,
+        "question": "Which genres have more than 100 tracks? List the genre name and count.",
+        "taxonomy": "Simple Join, Aggregation, Grouping, Having",
+        "ex_pass": true,
+        "esm_pass": false,
+        "generated_sql": "SELECT G.Name, COUNT(T.TrackId) AS TrackCount\nFROM Genre G\nJOIN Track T ON G.GenreId = T.GenreId\nGROUP BY G.GenreId\nHAVING COUNT(T.TrackId) > 100",
+        "gold_sql": "SELECT g.Name, COUNT(t.TrackId) as TrackCount FROM Genre g JOIN Track t ON g.GenreId = t.GenreId GROUP BY g.GenreId HAVING TrackCount > 100;"
+    },
+    {
+        "id": 14,
+        "question": "Calculate the average track length in seconds for each genre.",
+        "taxonomy": "Simple Join, Aggregation, Arithmetic, Grouping",
+        "ex_pass": false,
+        "esm_pass": false,
+        "generated_sql": "SELECT GenreId, AVG(Milliseconds / 1000.0) AS AverageLengthInSeconds FROM Track GROUP BY GenreId",
+        "gold_sql": "SELECT g.Name, AVG(t.Milliseconds) / 1000.0 as AvgSeconds FROM Genre g JOIN Track t ON g.GenreId = t.GenreId GROUP BY g.GenreId;"
+    },
+    {
+        "id": 15,
+        "question": "Identify the artist who has earned the most revenue from customers in Canada.",
+        "taxonomy": "Multi-Join, Aggregation, Grouping, Ordering, Limit",
+        "ex_pass": false,
+        "esm_pass": false,
+        "generated_sql": "SELECT T3.Name FROM Invoice AS T1 JOIN Customer AS T2 ON T1.CustomerId = T2.CustomerId JOIN InvoiceLine AS T4 ON T1.InvoiceId = T4.InvoiceId JOIN Track AS T5 ON T4.TrackId = T5.TrackId JOIN Album AS T6 ON T5.AlbumId = T6.AlbumId JOIN Artist AS T3 ON T6.ArtistId = T3.ArtistId WHERE T2.Country = 'Canada' GROUP BY T3.Name ORDER BY SUM(T1.Total) DESC LIMIT 1",
+        "gold_sql": "SELECT ar.Name, SUM(il.UnitPrice * il.Quantity) AS Revenue FROM Artist ar JOIN Album al ON ar.ArtistId = al.ArtistId JOIN Track t ON al.AlbumId = t.AlbumId JOIN InvoiceLine il ON t.TrackId = il.TrackId JOIN Invoice i ON il.InvoiceId = i.InvoiceId WHERE i.BillingCountry = 'Canada' GROUP BY ar.ArtistId ORDER BY Revenue DESC LIMIT 1;"
+    }
+]

backend/sql_eval_Qwen_Qwen2.5-Coder-7B-Instruct_featherless-ai.json

@@ -0,0 +1,137 @@
+[
+    {
+        "id": 1,
+        "question": "List all the artists name in the database.",
+        "taxonomy": "Selection",
+        "ex_pass": true,
+        "esm_pass": true,
+        "generated_sql": "SELECT Name FROM Artist",
+        "gold_sql": "SELECT Name FROM Artist;"
+    },
+    {
+        "id": 2,
+        "question": "How many genres are there?",
+        "taxonomy": "Aggregation",
+        "ex_pass": true,
+        "esm_pass": true,
+        "generated_sql": "SELECT COUNT(*) FROM Genre",
+        "gold_sql": "SELECT COUNT(*) FROM Genre;"
+    },
+    {
+        "id": 3,
+        "question": "List the names of the first 5 tracks.",
+        "taxonomy": "Selection, Limit",
+        "ex_pass": true,
+        "esm_pass": true,
+        "generated_sql": "SELECT Name FROM Track LIMIT 5",
+        "gold_sql": "SELECT Name FROM Track LIMIT 5;"
+    },
+    {
+        "id": 4,
+        "question": "Count the number of customers located in the USA.",
+        "taxonomy": "Aggregation, Filtering",
+        "ex_pass": true,
+        "esm_pass": true,
+        "generated_sql": "SELECT COUNT(*) FROM Customer WHERE Country = 'USA';",
+        "gold_sql": "SELECT COUNT(*) FROM Customer WHERE Country = 'USA';"
+    },
+    {
+        "id": 5,
+        "question": "Find all invoices for the customer with ID 1.",
+        "taxonomy": "Selection, Filtering",
+        "ex_pass": true,
+        "esm_pass": true,
+        "generated_sql": "SELECT * FROM Invoice WHERE CustomerId = 1;",
+        "gold_sql": "SELECT * FROM Invoice WHERE CustomerId = 1;"
+    },
+    {
+        "id": 6,
+        "question": "List each album title along with the artist's name.",
+        "taxonomy": "Simple Join",
+        "ex_pass": true,
+        "esm_pass": false,
+        "generated_sql": "SELECT T1.Title, T2.Name FROM Album AS T1 INNER JOIN Artist AS T2 ON T1.ArtistId = T2.ArtistId",
+        "gold_sql": "SELECT Album.Title, Artist.Name FROM Album JOIN Artist ON Album.ArtistId = Artist.ArtistId;"
+    },
+    {
+        "id": 7,
+        "question": "How many tracks belong to the 'Rock' genre?",
+        "taxonomy": "Simple Join, Filtering, Aggregation",
+        "ex_pass": true,
+        "esm_pass": true,
+        "generated_sql": "SELECT COUNT(*) FROM Track JOIN Genre ON Track.GenreId = Genre.GenreId WHERE Genre.Name = 'Rock'",
+        "gold_sql": "SELECT COUNT(*) FROM Track JOIN Genre ON Track.GenreId = Genre.GenreId WHERE Genre.Name = 'Rock';"
+    },
+    {
+        "id": 8,
+        "question": "Show the total revenue generated from each country.",
+        "taxonomy": "Aggregation, Grouping",
+        "ex_pass": true,
+        "esm_pass": false,
+        "generated_sql": "SELECT BillingCountry, SUM(Total) AS TotalRevenue FROM Invoice GROUP BY BillingCountry",
+        "gold_sql": "SELECT BillingCountry, SUM(Total) FROM Invoice GROUP BY BillingCountry;"
+    },
+    {
+        "id": 9,
+        "question": "Find the total number of items sold for each media type.",
+        "taxonomy": "Multi-Join, Aggregation, Grouping",
+        "ex_pass": false,
+        "esm_pass": false,
+        "generated_sql": "SELECT T2.Name, COUNT(T1.MediaTypeId) AS TotalItemsSold FROM SalesOrderDetail AS T1 INNER JOIN Media.MediaType AS T2 ON T1.MediaTypeId = T2.MediaTypeId GROUP BY T2.Name",
+        "gold_sql": "SELECT MediaType.Name, SUM(InvoiceLine.Quantity) FROM InvoiceLine JOIN Track ON InvoiceLine.TrackId = Track.TrackId JOIN MediaType ON Track.MediaTypeId = MediaType.MediaTypeId GROUP BY MediaType.Name;"
+    },
+    {
+        "id": 10,
+        "question": "List the first and last names of all employees who are Sales Support Agents.",
+        "taxonomy": "Selection, Filtering",
+        "ex_pass": true,
+        "esm_pass": true,
+        "generated_sql": "SELECT FirstName, LastName FROM Employee WHERE Title = 'Sales Support Agent'",
+        "gold_sql": "SELECT FirstName, LastName FROM Employee WHERE Title = 'Sales Support Agent';"
+    },
+    {
+        "id": 11,
+        "question": "List the top 5 customers who have spent the most money in total.",
+        "taxonomy": "Simple Join, Aggregation, Grouping, Ordering, Limit",
+        "ex_pass": false,
+        "esm_pass": false,
+        "generated_sql": "SELECT c.CustomerId, c.FirstName, c.LastName, SUM(i.Total) AS TotalSpent\nFROM Customer c\nJOIN Invoice i ON c.CustomerId = i.CustomerId\nGROUP BY c.CustomerId\nORDER BY TotalSpent DESC\nLIMIT 5",
+        "gold_sql": "SELECT c.FirstName, c.LastName, SUM(i.Total) as TotalSpent FROM Customer c JOIN Invoice i ON c.CustomerId = i.CustomerId GROUP BY c.CustomerId ORDER BY TotalSpent DESC LIMIT 5;"
+    },
+    {
+        "id": 12,
+        "question": "Which artist has the most tracks in the database? Give the name and count.",
+        "taxonomy": "Multi-Join, Aggregation, Grouping, Ordering, Limit",
+        "ex_pass": false,
+        "esm_pass": false,
+        "generated_sql": "SELECT T2.Name, COUNT(T1.AlbumId) AS TrackCount FROM Album AS T1 INNER JOIN Artist AS T2 ON T1.ArtistId = T2.ArtistId GROUP BY T2.Name ORDER BY TrackCount DESC LIMIT 1",
+        "gold_sql": "SELECT ar.Name, COUNT(t.TrackId) as TrackCount FROM Artist ar JOIN Album al ON ar.ArtistId = al.ArtistId JOIN Track t ON al.AlbumId = t.AlbumId GROUP BY ar.ArtistId ORDER BY TrackCount DESC LIMIT 1;"
+    },
+    {
+        "id": 13,
+        "question": "Which genres have more than 100 tracks? List the genre name and count.",
+        "taxonomy": "Simple Join, Aggregation, Grouping, Having",
+        "ex_pass": true,
+        "esm_pass": false,
+        "generated_sql": "SELECT T2.Name, COUNT(T1.TrackId) AS TrackCount FROM TRACK AS T1 INNER JOIN GENRE AS T2 ON T1.GenreId = T2.GenreId GROUP BY T2.Name HAVING COUNT(T1.TrackId) > 100",
+        "gold_sql": "SELECT g.Name, COUNT(t.TrackId) as TrackCount FROM Genre g JOIN Track t ON g.GenreId = t.GenreId GROUP BY g.GenreId HAVING TrackCount > 100;"
+    },
+    {
+        "id": 14,
+        "question": "Calculate the average track length in seconds for each genre.",
+        "taxonomy": "Simple Join, Aggregation, Arithmetic, Grouping",
+        "ex_pass": false,
+        "esm_pass": false,
+        "generated_sql": "SELECT AVG(Milliseconds / 1000) AS AverageLengthInSeconds, GenreId FROM Track GROUP BY GenreId",
+        "gold_sql": "SELECT g.Name, AVG(t.Milliseconds) / 1000.0 as AvgSeconds FROM Genre g JOIN Track t ON g.GenreId = t.GenreId GROUP BY g.GenreId;"
+    },
+    {
+        "id": 15,
+        "question": "Identify the artist who has earned the most revenue from customers in Canada.",
+        "taxonomy": "Multi-Join, Aggregation, Grouping, Ordering, Limit",
+        "ex_pass": false,
+        "esm_pass": false,
+        "generated_sql": "SELECT T1.Name FROM ARTIST AS T1 JOIN ALBUM AS T2 ON T1.ArtistId = T2.ArtistId JOIN TRACK AS T3 ON T2.AlbumId = T3.AlbumId JOIN INVOICE AS T4 ON T3.TrackId = T4.InvoiceLine.TrackId WHERE T4.InvoiceLine.CustomerId IN ( SELECT CustomerId FROM CUSTOMER WHERE Country = 'Canada' ) GROUP BY T1.Name ORDER BY SUM(T4.InvoiceLine.UnitPrice * T4.InvoiceLine.Quantity) DESC LIMIT 1",
+        "gold_sql": "SELECT ar.Name, SUM(il.UnitPrice * il.Quantity) AS Revenue FROM Artist ar JOIN Album al ON ar.ArtistId = al.ArtistId JOIN Track t ON al.AlbumId = t.AlbumId JOIN InvoiceLine il ON t.TrackId = il.TrackId JOIN Invoice i ON il.InvoiceId = i.InvoiceId WHERE i.BillingCountry = 'Canada' GROUP BY ar.ArtistId ORDER BY Revenue DESC LIMIT 1;"
+    }
+]

backend/sql_eval_defog_llama-3-sqlcoder-8b_featherless-ai.json

@@ -0,0 +1,137 @@
+[
+    {
+        "id": 1,
+        "question": "List all the artists name in the database.",
+        "taxonomy": "Selection",
+        "ex_pass": true,
+        "esm_pass": false,
+        "generated_sql": "SELECT a.Name FROM Artist a;",
+        "gold_sql": "SELECT Name FROM Artist;"
+    },
+    {
+        "id": 2,
+        "question": "How many genres are there?",
+        "taxonomy": "Aggregation",
+        "ex_pass": true,
+        "esm_pass": true,
+        "generated_sql": "SELECT COUNT(*) FROM Genre;",
+        "gold_sql": "SELECT COUNT(*) FROM Genre;"
+    },
+    {
+        "id": 3,
+        "question": "List the names of the first 5 tracks.",
+        "taxonomy": "Selection, Limit",
+        "ex_pass": true,
+        "esm_pass": false,
+        "generated_sql": "SELECT t.Name FROM Track t ORDER BY t.TrackId LIMIT 5;",
+        "gold_sql": "SELECT Name FROM Track LIMIT 5;"
+    },
+    {
+        "id": 4,
+        "question": "Count the number of customers located in the USA.",
+        "taxonomy": "Aggregation, Filtering",
+        "ex_pass": true,
+        "esm_pass": false,
+        "generated_sql": "SELECT COUNT(*) FROM Customer c WHERE c.Country = 'USA';",
+        "gold_sql": "SELECT COUNT(*) FROM Customer WHERE Country = 'USA';"
+    },
+    {
+        "id": 5,
+        "question": "Find all invoices for the customer with ID 1.",
+        "taxonomy": "Selection, Filtering",
+        "ex_pass": false,
+        "esm_pass": false,
+        "generated_sql": "SELECT i.InvoiceId, i.InvoiceDate, i.Total FROM Invoice i WHERE i.CustomerId = 1;",
+        "gold_sql": "SELECT * FROM Invoice WHERE CustomerId = 1;"
+    },
+    {
+        "id": 6,
+        "question": "List each album title along with the artist's name.",
+        "taxonomy": "Simple Join",
+        "ex_pass": true,
+        "esm_pass": false,
+        "generated_sql": "SELECT a.Title, ar.Name FROM Album a JOIN Artist ar ON a.ArtistId = ar.ArtistId;",
+        "gold_sql": "SELECT Album.Title, Artist.Name FROM Album JOIN Artist ON Album.ArtistId = Artist.ArtistId;"
+    },
+    {
+        "id": 7,
+        "question": "How many tracks belong to the 'Rock' genre?",
+        "taxonomy": "Simple Join, Filtering, Aggregation",
+        "ex_pass": true,
+        "esm_pass": false,
+        "generated_sql": "SELECT COUNT(t.TrackId) FROM Track t JOIN Genre g ON t.GenreId = g.GenreId WHERE g.Name = 'Rock';",
+        "gold_sql": "SELECT COUNT(*) FROM Track JOIN Genre ON Track.GenreId = Genre.GenreId WHERE Genre.Name = 'Rock';"
+    },
+    {
+        "id": 8,
+        "question": "Show the total revenue generated from each country.",
+        "taxonomy": "Aggregation, Grouping",
+        "ex_pass": true,
+        "esm_pass": false,
+        "generated_sql": "SELECT i.BillingCountry, SUM(i.Total) AS TotalRevenue FROM Invoice i GROUP BY i.BillingCountry ORDER BY TotalRevenue DESC NULLS LAST;",
+        "gold_sql": "SELECT BillingCountry, SUM(Total) FROM Invoice GROUP BY BillingCountry;"
+    },
+    {
+        "id": 9,
+        "question": "Find the total number of items sold for each media type.",
+        "taxonomy": "Multi-Join, Aggregation, Grouping",
+        "ex_pass": false,
+        "esm_pass": false,
+        "generated_sql": "SELECT mt.Name, COUNT(*) AS total_items_sold FROM MediaType mt JOIN (SELECT MediaType.MediaTypeId, COUNT(*) AS items_sold FROM Album JOIN MediaType ON Album.MediaTypeId = MediaType.MediaTypeId GROUP BY MediaType.MediaTypeId) AS ms ON mt.MediaTypeId = ms.MediaTypeId GROUP BY mt.Name ORDER BY total_items_sold DESC NULLS LAST;",
+        "gold_sql": "SELECT MediaType.Name, SUM(InvoiceLine.Quantity) FROM InvoiceLine JOIN Track ON InvoiceLine.TrackId = Track.TrackId JOIN MediaType ON Track.MediaTypeId = MediaType.MediaTypeId GROUP BY MediaType.Name;"
+    },
+    {
+        "id": 10,
+        "question": "List the first and last names of all employees who are Sales Support Agents.",
+        "taxonomy": "Selection, Filtering",
+        "ex_pass": true,
+        "esm_pass": false,
+        "generated_sql": "SELECT e.FirstName, e.LastName FROM Employee e WHERE e.Title = 'Sales Support Agent';",
+        "gold_sql": "SELECT FirstName, LastName FROM Employee WHERE Title = 'Sales Support Agent';"
+    },
+    {
+        "id": 11,
+        "question": "List the top 5 customers who have spent the most money in total.",
+        "taxonomy": "Simple Join, Aggregation, Grouping, Ordering, Limit",
+        "ex_pass": false,
+        "esm_pass": false,
+        "generated_sql": "SELECT c.CustomerId, SUM(o.TotalAmount) AS TotalSpent FROM Customer c JOIN Order o ON c.CustomerId = o.CustomerId GROUP BY c.CustomerId ORDER BY TotalSpent DESC LIMIT 5;",
+        "gold_sql": "SELECT c.FirstName, c.LastName, SUM(i.Total) as TotalSpent FROM Customer c JOIN Invoice i ON c.CustomerId = i.CustomerId GROUP BY c.CustomerId ORDER BY TotalSpent DESC LIMIT 5;"
+    },
+    {
+        "id": 12,
+        "question": "Which artist has the most tracks in the database? Give the name and count.",
+        "taxonomy": "Multi-Join, Aggregation, Grouping, Ordering, Limit",
+        "ex_pass": false,
+        "esm_pass": false,
+        "generated_sql": "SELECT a.\"ArtistId\", COUNT(a.\"AlbumId\") AS track_count FROM \"Album\" a GROUP BY a.\"ArtistId\" ORDER BY track_count DESC LIMIT 1;",
+        "gold_sql": "SELECT ar.Name, COUNT(t.TrackId) as TrackCount FROM Artist ar JOIN Album al ON ar.ArtistId = al.ArtistId JOIN Track t ON al.AlbumId = t.AlbumId GROUP BY ar.ArtistId ORDER BY TrackCount DESC LIMIT 1;"
+    },
+    {
+        "id": 13,
+        "question": "Which genres have more than 100 tracks? List the genre name and count.",
+        "taxonomy": "Simple Join, Aggregation, Grouping, Having",
+        "ex_pass": true,
+        "esm_pass": false,
+        "generated_sql": "SELECT g.Name, COUNT(t.TrackId) AS TrackCount FROM Genre g JOIN Track t ON g.GenreId = t.GenreId GROUP BY g.Name HAVING COUNT(t.TrackId) > 100 ORDER BY TrackCount DESC NULLS LAST;",
+        "gold_sql": "SELECT g.Name, COUNT(t.TrackId) as TrackCount FROM Genre g JOIN Track t ON g.GenreId = t.GenreId GROUP BY g.GenreId HAVING TrackCount > 100;"
+    },
+    {
+        "id": 14,
+        "question": "Calculate the average track length in seconds for each genre.",
+        "taxonomy": "Simple Join, Aggregation, Arithmetic, Grouping",
+        "ex_pass": false,
+        "esm_pass": false,
+        "generated_sql": "SELECT t.GenreId, AVG(t.Milliseconds / 1000.0) AS average_length FROM Track t GROUP BY t.GenreId ORDER BY t.GenreId NULLS LAST;",
+        "gold_sql": "SELECT g.Name, AVG(t.Milliseconds) / 1000.0 as AvgSeconds FROM Genre g JOIN Track t ON g.GenreId = t.GenreId GROUP BY g.GenreId;"
+    },
+    {
+        "id": 15,
+        "question": "Identify the artist who has earned the most revenue from customers in Canada.",
+        "taxonomy": "Multi-Join, Aggregation, Grouping, Ordering, Limit",
+        "ex_pass": false,
+        "esm_pass": false,
+        "generated_sql": "SELECT a.Name, SUM(al.Sales) AS TotalRevenue FROM Artist a JOIN Album al ON a.ArtistId = al.ArtistId JOIN Customer c ON al.CustomerId = c.CustomerId WHERE c.Country = 'Canada' GROUP BY a.Name ORDER BY TotalRevenue DESC LIMIT 1;",
+        "gold_sql": "SELECT ar.Name, SUM(il.UnitPrice * il.Quantity) AS Revenue FROM Artist ar JOIN Album al ON ar.ArtistId = al.ArtistId JOIN Track t ON al.AlbumId = t.AlbumId JOIN InvoiceLine il ON t.TrackId = il.TrackId JOIN Invoice i ON il.InvoiceId = i.InvoiceId WHERE i.BillingCountry = 'Canada' GROUP BY ar.ArtistId ORDER BY Revenue DESC LIMIT 1;"
+    }
+]

backend/sql_eval_defog_sqlcoder-7b-2.json

@@ -0,0 +1,137 @@
+[
+    {
+        "id": 1,
+        "question": "List all the artists name in the database.",
+        "taxonomy": "Selection",
+        "ex_pass": true,
+        "esm_pass": false,
+        "generated_sql": "SELECT a.\"Name\" FROM \"Artist\" a;",
+        "gold_sql": "SELECT Name FROM Artist;"
+    },
+    {
+        "id": 2,
+        "question": "How many genres are there?",
+        "taxonomy": "Aggregation",
+        "ex_pass": true,
+        "esm_pass": false,
+        "generated_sql": "SELECT COUNT(g.genreid) AS number_of_genres FROM genre g;",
+        "gold_sql": "SELECT COUNT(*) FROM Genre;"
+    },
+    {
+        "id": 3,
+        "question": "List the names of the first 5 tracks.",
+        "taxonomy": "Selection, Limit",
+        "ex_pass": true,
+        "esm_pass": false,
+        "generated_sql": "SELECT t.name FROM \"Track\" t LIMIT 5;",
+        "gold_sql": "SELECT Name FROM Track LIMIT 5;"
+    },
+    {
+        "id": 4,
+        "question": "Count the number of customers located in the USA.",
+        "taxonomy": "Aggregation, Filtering",
+        "ex_pass": false,
+        "esm_pass": false,
+        "generated_sql": "SELECT COUNT(*) FROM \"Customer\" c WHERE c.Country ILIKE '%USA%';",
+        "gold_sql": "SELECT COUNT(*) FROM Customer WHERE Country = 'USA';"
+    },
+    {
+        "id": 5,
+        "question": "Find all invoices for the customer with ID 1.",
+        "taxonomy": "Selection, Filtering",
+        "ex_pass": true,
+        "esm_pass": false,
+        "generated_sql": "SELECT * FROM Invoice i WHERE i.CustomerId = 1;",
+        "gold_sql": "SELECT * FROM Invoice WHERE CustomerId = 1;"
+    },
+    {
+        "id": 6,
+        "question": "List each album title along with the artist's name.",
+        "taxonomy": "Simple Join",
+        "ex_pass": true,
+        "esm_pass": false,
+        "generated_sql": "SELECT a.\"Title\", ar.\"Name\" AS \"Artist Name\" FROM \"Album\" a JOIN \"Artist\" ar ON a.\"ArtistId\" = ar.\"ArtistId\";",
+        "gold_sql": "SELECT Album.Title, Artist.Name FROM Album JOIN Artist ON Album.ArtistId = Artist.ArtistId;"
+    },
+    {
+        "id": 7,
+        "question": "How many tracks belong to the 'Rock' genre?",
+        "taxonomy": "Simple Join, Filtering, Aggregation",
+        "ex_pass": true,
+        "esm_pass": false,
+        "generated_sql": "SELECT COUNT(t.TrackId) AS number_of_tracks FROM \"Track\" t JOIN \"Genre\" g ON t.GenreId = g.GenreId WHERE g.Name = 'Rock';",
+        "gold_sql": "SELECT COUNT(*) FROM Track JOIN Genre ON Track.GenreId = Genre.GenreId WHERE Genre.Name = 'Rock';"
+    },
+    {
+        "id": 8,
+        "question": "Show the total revenue generated from each country.",
+        "taxonomy": "Aggregation, Grouping",
+        "ex_pass": false,
+        "esm_pass": false,
+        "generated_sql": "SELECT c.country_name, SUM(i.total) AS total_revenue FROM \"Invoice\" i JOIN \"Customer\" c ON i.customer_id = c.customer_id GROUP BY c.country_name ORDER BY total_revenue DESC NULLS LAST;",
+        "gold_sql": "SELECT BillingCountry, SUM(Total) FROM Invoice GROUP BY BillingCountry;"
+    },
+    {
+        "id": 9,
+        "question": "Find the total number of items sold for each media type.",
+        "taxonomy": "Multi-Join, Aggregation, Grouping",
+        "ex_pass": false,
+        "esm_pass": false,
+        "generated_sql": "SELECT m.name AS media_type, COUNT(*) AS total_items_sold FROM \"MediaType\" m JOIN \"Sales\" s ON m.media_type_id = s.media_type_id GROUP BY m.name;",
+        "gold_sql": "SELECT MediaType.Name, SUM(InvoiceLine.Quantity) FROM InvoiceLine JOIN Track ON InvoiceLine.TrackId = Track.TrackId JOIN MediaType ON Track.MediaTypeId = MediaType.MediaTypeId GROUP BY MediaType.Name;"
+    },
+    {
+        "id": 10,
+        "question": "List the first and last names of all employees who are Sales Support Agents.",
+        "taxonomy": "Selection, Filtering",
+        "ex_pass": false,
+        "esm_pass": false,
+        "generated_sql": "SELECT e.first_name, e.last_name FROM \"Employee\" e WHERE e.title = 'Sales Support Agent';",
+        "gold_sql": "SELECT FirstName, LastName FROM Employee WHERE Title = 'Sales Support Agent';"
+    },
+    {
+        "id": 11,
+        "question": "List the top 5 customers who have spent the most money in total.",
+        "taxonomy": "Simple Join, Aggregation, Grouping, Ordering, Limit",
+        "ex_pass": false,
+        "esm_pass": false,
+        "generated_sql": "SELECT c.customerid, c.firstname, c.lastname, SUM(oi.productid * oi.quantity) AS total_spent FROM customer c JOIN orders o ON c.customerid = o.customerid JOIN orderitems oi ON o.orderid = oi.orderid GROUP BY c.customerid, c.firstname, c.lastname ORDER BY total_spent DESC LIMIT 5;",
+        "gold_sql": "SELECT c.FirstName, c.LastName, SUM(i.Total) as TotalSpent FROM Customer c JOIN Invoice i ON c.CustomerId = i.CustomerId GROUP BY c.CustomerId ORDER BY TotalSpent DESC LIMIT 5;"
+    },
+    {
+        "id": 12,
+        "question": "Which artist has the most tracks in the database? Give the name and count.",
+        "taxonomy": "Multi-Join, Aggregation, Grouping, Ordering, Limit",
+        "ex_pass": false,
+        "esm_pass": false,
+        "generated_sql": "",
+        "gold_sql": "SELECT ar.Name, COUNT(t.TrackId) as TrackCount FROM Artist ar JOIN Album al ON ar.ArtistId = al.ArtistId JOIN Track t ON al.AlbumId = t.AlbumId GROUP BY ar.ArtistId ORDER BY TrackCount DESC LIMIT 1;"
+    },
+    {
+        "id": 13,
+        "question": "Which genres have more than 100 tracks? List the genre name and count.",
+        "taxonomy": "Simple Join, Aggregation, Grouping, Having",
+        "ex_pass": true,
+        "esm_pass": false,
+        "generated_sql": "SELECT g.name, COUNT(t.trackid) AS COUNT FROM genre g JOIN track t ON g.genreid = t.genreid GROUP BY g.name HAVING COUNT(t.trackid) > 100 ORDER BY COUNT DESC NULLS LAST;",
+        "gold_sql": "SELECT g.Name, COUNT(t.TrackId) as TrackCount FROM Genre g JOIN Track t ON g.GenreId = t.GenreId GROUP BY g.GenreId HAVING TrackCount > 100;"
+    },
+    {
+        "id": 14,
+        "question": "Calculate the average track length in seconds for each genre.",
+        "taxonomy": "Simple Join, Aggregation, Arithmetic, Grouping",
+        "ex_pass": false,
+        "esm_pass": false,
+        "generated_sql": "SELECT g.Name, AVG(t.Milliseconds / 1000)) AS AverageDuration FROM Track t JOIN Genre g ON t.GenreId = g.GenreId GROUP BY g.Name ORDER BY AverageDuration DESC NULLS LAST;",
+        "gold_sql": "SELECT g.Name, AVG(t.Milliseconds) / 1000.0 as AvgSeconds FROM Genre g JOIN Track t ON g.GenreId = t.GenreId GROUP BY g.GenreId;"
+    },
+    {
+        "id": 15,
+        "question": "Identify the artist who has earned the most revenue from customers in Canada.",
+        "taxonomy": "Multi-Join, Aggregation, Grouping, Ordering, Limit",
+        "ex_pass": false,
+        "esm_pass": false,
+        "generated_sql": "SELECT a.\"Name\" FROM \"Artist\" a JOIN \"Album\" al ON a.\"ArtistId\" = al.\"AlbumArtistId\" WHERE al.\"AlbumCountry\" = 'Canada' ORDER BY a.\"Name\" DESC NULLS LAST LIMIT 1;",
+        "gold_sql": "SELECT ar.Name, SUM(il.UnitPrice * il.Quantity) AS Revenue FROM Artist ar JOIN Album al ON ar.ArtistId = al.ArtistId JOIN Track t ON al.AlbumId = t.AlbumId JOIN InvoiceLine il ON t.TrackId = il.TrackId JOIN Invoice i ON il.InvoiceId = i.InvoiceId WHERE i.BillingCountry = 'Canada' GROUP BY ar.ArtistId ORDER BY Revenue DESC LIMIT 1;"
+    }
+]

backend/src/scripts/evaluation_mode.py

@@ -5,53 +5,79 @@ import sqlglot
 from pathlib import Path
 import pandas as pd
 from src.database.db_manager import get_db_connection
+from src.nl2sql.hf_engine import get_models
 from src.nl2sql.sql_agent import nl2sql_agent
 from src.scripts.taxonomy_report import print_taxonomyReport
 
 TEST_CASES_PATH = Path("src/scripts/test_cases.json")
-RESULTS_PATH = Path("hf_evaluation_results.json")
 
-def _normalize_dataframe(dataframe: pd.DataFrame) -> pd.DataFrame:
+def _normalize_dataframe(dataframe: pd.DataFrame) -> list:
     # Normalize dataframe to ensure accurate comparison
     """
     Standardize dataframes for Execution Accuracy (EX).
-    - Ensures Order Agnoticism by sorting all values.
-    - Prepares for Column Agnoticism by focuing on value comparison rather than column names.
+    - Converts dataframe to a list of tuples to ignore column names.
+    - Rounds floating points to 4 decimal places to avoid precision mismatch.
+    - Sorts the final list to ensure order-agnostic comparison.
     """
+    if dataframe is None or dataframe.empty:
+        return []
+    
     normalized = dataframe.copy()
-    #normalized.columns = [str(column).lower() for column in normalized.columns]
 
     for column in normalized.columns:
         normalized[column] = normalized[column].map(
-            lambda value: round(float(value), 6)
-            if isinstance(value, (float, int))
-            else value
+            lambda x: round(float(x), 4)
+            if pd.api.types.is_numeric_dtype(type(x)) and isinstance(x, float)
+            else x
+            #lambda value: round(float(value), 6)
+            #if isinstance(value, (float, int))
+            #else value
         )
 
-    sort_columns = list(normalized.columns)
-    if sort_columns:
-        normalized = normalized.sort_values(by=sort_columns).reset_index(drop=True)
+    # Convert to list of tuples for order-agnostic comparison
+    data_tuples = [tuple(row) for row in normalized.to_numpy()]
 
-    return normalized
+    # Sort to ensure order agnoticism
+    try:
+        data_tuples.sort(key=lambda x: str(x))
+    except Exception as e:
+        pass
+        
+    return data_tuples
+
+# Semantic safety net
+def extract_tables(sql: str) -> set:
+    """
+    Extract a set of all table names used in a SQL query.
+    Used to catch false positives where EX passes but the model queried the wrong tables.
+    """
+    if not sql:
+        return set()
+    try:
+        parsed = sqlglot.parse_one(sql, read=None)
+
+        # Find all table expressions & extract names, ignore aliases
+        return set(table.name.lower() for table in parsed.find_all(sqlglot.exp.Table) if table.name)
+    except Exception as e:
+        return set()
 
 # EX: Compare generated SQL results with expected results
 def calculate_ex(df_generated: pd.DataFrame, df_gold: pd.DataFrame) -> bool:
     """
     Execution Accuracy (EX): Compare generated SQL results with expected results.
-    - Column Name Agnostic: Use .values to ignore header differences.
     """
     if df_generated is None or df_gold is None:
         return False
 
+    #if normalized_generated.shape != normalized_gold.shape:
+    #        return False
+    
     try:
         normalized_generated = _normalize_dataframe(df_generated)
         normalized_gold = _normalize_dataframe(df_gold)
-
-        if normalized_generated.shape != normalized_gold.shape:
-            return False
         
-        return bool((normalized_generated.values == normalized_gold.values).all())
-        # return normalized_generated.equals(normalized_gold)
+        return normalized_generated == normalized_gold
+
     except Exception as error:
         print(f"EX Evaluation Error: {error}")
         return False
@@ -61,6 +87,9 @@ def calculate_esm(generated_sql: str, gold_sql: str) -> bool:
     Exact Set Match (ESM): Compare AST structure using sqlglot.
     - Ignores formatting, capitalization, and minor syntactic sugar.
     """
+    if not generated_sql or not gold_sql:
+        return False
+    
     try:
         # Parse both SQL queries into expressions
         generated_exp = sqlglot.parse_one(generated_sql, read=None)
@@ -72,7 +101,10 @@ def calculate_esm(generated_sql: str, gold_sql: str) -> bool:
         print(f"ESM Evaluation Error: {error}")
         return False
 
-def run_evaluation():
+def run_evaluation(model_id: str):
+    print(f"\nRunning SQL evaluation for model: {model_id}")
+    print("\n" + "-" *50)
+
     if not TEST_CASES_PATH.exists():
         print(f"Error: Could not find test cases at {TEST_CASES_PATH}")
         return
@@ -91,10 +123,10 @@ def run_evaluation():
         question = case.get("question")
         gold_sql = case.get("gold_sql")
         taxonomy = case.get("taxonomy", "Unknown")
-        # print(f"Testing ID {id}: {question[:50]}...")
+        print(f"Testing ID {id}: {question[:40]}...")
 
         # Implement agent to handle RAG retrieval and SQL generation
-        agent_response = nl2sql_agent(user_question=question)
+        agent_response = nl2sql_agent(user_question=question, model_id=model_id)
         generated_sql = agent_response.get("query", "")
 
         # ESM Evaluation
@@ -112,7 +144,22 @@ def run_evaluation():
             df_generated = pd.read_sql_query(generated_sql, connection)
             df_gold = pd.read_sql_query(gold_sql, connection)
 
+            # Trap the False Positive (empty set): weak test case
+            if df_gold.empty:
+                print(f"[!]WARNING: Gold SQL for ID {id} returned an emoty response.")
+
             ex_result = calculate_ex(df_generated, df_gold)
+
+            # Semantic safety net check
+            if ex_result:
+                gen_tables = extract_tables(generated_sql)
+                gold_tables = extract_tables(gold_sql)
+
+                if gen_tables != gold_tables:
+                    print(f"[X] FALSE POSITIVE (ID{id}): Data matched, tables not")
+                    print(f"\nGenerated SQL tables: {gen_tables} | Gold SQL tables: {gold_tables}")
+                    ex_result = False
+        
             if ex_result:
                 ex_count += 1
         except Exception as error:
@@ -137,11 +184,22 @@ def run_evaluation():
 
     print("\nEVALUATION SUMMARY")
     print("-" * 40)
+    print(f"Model Evaluated: {model_id}")
     print(f"Total Test Cases: {total}")
     print(f"Execution Accuracy (EX): {ex_accuracy:.2f}% ({ex_count}/{total})")
     print(f"Exact Set Match (ESM): {esm_accuracy:.2f}% ({esm_count}/{total})")
 
-    with RESULTS_PATH.open("w", encoding="utf-8") as handle:
+    safe_model_name = model_id.replace("/", "_").replace(":", "_")
+    output_file = Path(f"sql_eval_{safe_model_name}.json")
+    with output_file.open("w", encoding="utf-8") as handle:
         json.dump(results, handle, indent=4)
 
-    print_taxonomyReport(results)
+    print_taxonomyReport(results)
+
+if __name__ == "__main__":
+    from dotenv import load_dotenv, find_dotenv
+    load_dotenv(find_dotenv())
+
+    models_to_test = get_models()
+    for model in models_to_test:
+        run_evaluation(model)

backend/src/scripts/interactive_mode.py

@@ -1,43 +1,129 @@
 # Path: src/scripts/interactive_mode.py
 # Interactive mode: Allows user to manually type questions and see the agent's response
+import csv
+import json
+import pandas as pd
+from pathlib import Path
 from tabulate import tabulate
+from src.database.db_manager import get_db_connection
 from src.nl2sql.sql_agent import nl2sql_agent
+from src.nl2sql.hf_engine import get_models
 
-def run_interactiveMode():
-    """ Allows user to manually type questions and get agent's response."""
+TEST_CASES_PATH = Path("src/scripts/test_cases.json")
+
+def get_query_data(sql_query: str) -> pd.DataFrame:
+    """
+    Executes a SQL query and returns the results as a DataFrame.
+    """
+    if not sql_query or sql_query == "N/A":
+        return pd.DataFrame()
+    
+    connection = get_db_connection()
+    if not connection:
+        return pd.DataFrame()
+
+    try:
+        df = pd.read_sql_query(sql_query, connection)
+        return df
+    except Exception as e:
+        print(f"Error executing SQL query: {e}")
+        return pd.DataFrame()
+    finally:
+        connection.close()
+
+def verify_data(df_gold: pd.DataFrame, df_generated: pd.DataFrame) -> bool:
+    """
+    Guardrail check:
+    Verifies if the generated DataFrame matches the expected gold DataFrame.
+    """
+    if df_gold.empty and df_generated.empty:
+        return False  # Both empty: To catch this as a potential issue
+    
+    if len(df_gold) != len(df_generated):
+        return False
+    
+    try:
+        gold_val = df_gold.fillna("").astype(str).values.tolist()
+        gen_val = df_generated.fillna("").astype(str).values.tolist()
+
+        # Strip whitespace and convert to tuples (sortable)
+        gold_tuples = [tuple(val.strip() for val in row) for row in gold_val]
+        gen_tuples = [tuple(val.strip() for val in row) for row in gen_val]
+
+        return sorted(gold_tuples) == sorted(gen_tuples)
+    except Exception as e:
+        print(f"Error during data verification: {e}")
+        return False
+
+
+def run_interactiveMode(model_id: str):
+    """ 
+    Automates the interactive Question Answering mode.
+    Runs predefined questions through the agent and logs the textual NL response.
+    """
     print("\n========= Interactive NL2SQL Mode =========")
-    print("Type 'exit' or 'q' to return to the main menu.\n")
-
-    while True:
-        question = input("\nEnter your question: ")
-        if question.lower() in ['exit', 'q']:
-            break
-        if not question.strip():
-            continue
-
-        print("\nProcessing your question...")
-        response = nl2sql_agent(question)
-
-        print("\n========= Agent Response =========")
-        if response.get('status') == 'success':
-            print(f"Answer: {response.get('nl_response')}\n")
-
-            raw_data = response.get('results')
-            if raw_data:
-                display_data = raw_data
-                #display_data = raw_data[:20]
-                # Format data returned as a grid table
-                table = tabulate(display_data, tablefmt="grid")
-
-                print("Data Table:")
-                print(table)
-                print()
-            else:
-                print("No data returned from the database.")
-            
-            print(f"Generated SQL:\n{response.get('query')}\n")
-        else:
-            print(f"Status: {response.get('status')}")
-            print(f"Generated SQL:\n{response.get('query')}")
-            print(f"\nError Details:\n{response.get('error')}")
-        print("==================================\n")
+    print(f"Running Interactive Question Answering Evaluation on Model: {model_id}")
+    #print("Type 'exit' or 'q' to return to the main menu.\n")
+
+    if not TEST_CASES_PATH.exists():
+        print(f"Error: Could not find test cases at {TEST_CASES_PATH}")
+        return
+    
+    with TEST_CASES_PATH.open("r", encoding="utf-8") as handle:
+        test_cases = json.load(handle)
+
+    questAns_results = []
+
+    for case in test_cases:
+        case_id = case.get("id")
+        question = case.get("question")
+        gold_sql = case.get("gold_sql")
+        print(f"\n\nTesting ID {case_id}: {question[:40]}...")
+
+        response = nl2sql_agent(user_question=question, model_id=model_id)
+
+        # Extract metadata
+        status = response.get('status')
+        nl_answer = response.get('nl_response', 'N/A')
+        sql_query = response.get('query', 'N/A')
+        error_msg = response.get('error', '')
+        attempts = response.get('attempts', 0)
+
+        # Data cross-check
+        df_gold = get_query_data(gold_sql)
+        df_generated = get_query_data(sql_query)
+
+        # Verify accuracy
+        is_data_accurate = verify_data(df_gold, df_generated)
+
+        questAns_results.append({
+            "id": case_id,
+            "model_id": model_id,
+            "question": question,
+            "status": status,
+            "data_returned_correct": is_data_accurate,
+            "attempts": attempts,
+            "nl_response": nl_answer,
+            "sql_generated": sql_query,
+            "error": error_msg
+        })
+
+    # Save to CSV for human-readable and easy comparison
+    safe_model_name = model_id.replace("/", "_").replace(":", "_").replace(" ", "_")
+    output_csv = Path(f"Q&A_report_{safe_model_name}.csv")
+
+    keys = questAns_results[0].keys()
+    with output_csv.open("w", newline='', encoding="utf-8") as f:
+        dict_writer = csv.DictWriter(f, fieldnames=keys)
+        dict_writer.writeheader()
+        dict_writer.writerows(questAns_results)
+
+    print(f"\nInteractive evaluation completed. Results saved to: {output_csv}")
+
+if __name__ == "__main__":
+    from dotenv import load_dotenv, find_dotenv
+    load_dotenv(find_dotenv())
+
+    models_to_test = get_models()
+    for model in models_to_test:
+        run_interactiveMode(model)