NL2SQL Qwen Model V1.0.0

evaluation_results.json

@@ -0,0 +1,72 @@
+[
+    {
+        "id": 1,
+        "status": "ERROR: Execution failed on sql 'SELECT t.Genre, COUNT(t.TrackId) AS TrackCount FROM Track t GROUP BY t.Genre;': no such column: t.Genre"
+    },
+    {
+        "id": 2,
+        "status": "PASS",
+        "generated": "SELECT t1.Title, ar.Name FROM Album AS t1 INNER JOIN Artist AS ar ON t1.ArtistId = ar.ArtistId;"
+    },
+    {
+        "id": 3,
+        "status": "ERROR: Execution failed on sql 'SELECT T4.Country, SUM(T3.UnitPrice * T3.Quantity) as TotalRevenue\nFROM InvoiceLine AS T1\nJOIN Track AS T2 ON T1.TrackId = T2.TrackId\nJOIN Album AS T3 ON T2.AlbumId = T3.AlbumId\nJOIN Genre AS T4 ON T2.GenreId = T4.GenreId\nGROUP BY T4.Country;': no such column: T4.Country"
+    },
+    {
+        "id": 4,
+        "status": "ERROR: Execution failed on sql 'SELECT e.FirstName, e.LastName FROM Employee AS e JOIN Customer AS c ON e.SupportRepId = c.CustomerId WHERE c.Company LIKE 'Sales Support';': no such column: e.SupportRepId"
+    },
+    {
+        "id": 5,
+        "status": "FAIL",
+        "generated": "SELECT c.FirstName, c.LastName, SUM(il.UnitPrice * il.Quantity) as TotalSpent\nFROM Customer c\nJOIN Invoice i ON c.CustomerId = i.CustomerId\nJOIN InvoiceLine il ON i.InvoiceId = il.InvoiceId\nGROUP BY c.CustomerId\nORDER BY TotalSpent DESC\nLIMIT 5;"
+    },
+    {
+        "id": 6,
+        "status": "FAIL",
+        "generated": "SELECT T3.Name AS ArtistName, T1.Name AS SongName\nFROM Track AS T1\nJOIN Album AS T2 ON T1.AlbumId = T2.AlbumId\nJOIN Genre AS T3 ON T1.GenreId = T3.GenreId\nWHERE T3.Name = 'Rock'\nGROUP BY T1.TrackId;"
+    },
+    {
+        "id": 7,
+        "status": "ERROR: Execution failed on sql 'SELECT T3.Name AS MediaTypeName, COUNT(T1.TrackId) AS TotalTracksSold\nFROM Track AS T1\nJOIN MediaType AS T2 ON T1.MediaTypeId = T2.MediaTypeId\nGROUP BY T2.Name;': no such column: T3.Name"
+    },
+    {
+        "id": 8,
+        "status": "PASS",
+        "generated": "SELECT T3.Name FROM Playlist AS T1 INNER JOIN PlaylistTrack AS T2 ON T1.PlaylistId = T2.PlaylistId INNER JOIN Track AS T3 ON T2.TrackId = T3.TrackId WHERE T1.Name = 'TV Shows';"
+    },
+    {
+        "id": 9,
+        "status": "PASS",
+        "generated": "SELECT a.Name, COUNT(t.TrackId) AS TrackCount\nFROM Artist a\nJOIN Album ar ON a.ArtistId = ar.ArtistId\nJOIN Track t ON ar.AlbumId = t.AlbumId\nGROUP BY a.ArtistId\nORDER BY TrackCount DESC\nLIMIT 1;"
+    },
+    {
+        "id": 10,
+        "status": "FAIL",
+        "generated": "SELECT g.Name FROM Genre g JOIN Track t ON g.GenreId = t.GenreId GROUP BY g.Name HAVING COUNT(t.TrackId) > 100;"
+    },
+    {
+        "id": 11,
+        "status": "FAIL",
+        "generated": "SELECT a.Name, SUM(il.UnitPrice * il.Quantity) as TotalRevenue\nFROM Artist a\nJOIN Album al ON a.ArtistId = al.ArtistId\nJOIN Track tr ON al.AlbumId = tr.AlbumId\nJOIN InvoiceLine il ON tr.TrackId = il.TrackId\nGROUP BY a.ArtistId\nORDER BY TotalRevenue DESC\nLIMIT 1;"
+    },
+    {
+        "id": 12,
+        "status": "FAIL",
+        "generated": "SELECT a.Title, AVG(t.Milliseconds) / 1000 AS AverageLengthInSeconds\nFROM Album a\nJOIN Track t ON a.AlbumId = t.AlbumId\nGROUP BY a.AlbumId;"
+    },
+    {
+        "id": 13,
+        "status": "PASS",
+        "generated": "SELECT c.FirstName, c.LastName, e.FirstName, e.LastName\nFROM Customer c\nJOIN Employee e ON c.SupportRepId = e.EmployeeId\nWHERE e.LastName = 'Peacock';"
+    },
+    {
+        "id": 14,
+        "status": "PASS",
+        "generated": "SELECT t2.City FROM Invoice AS t1 INNER JOIN Customer AS t2 ON t1.CustomerId = t2.CustomerId WHERE strftime('%Y', t1.InvoiceDate) = '2013' GROUP BY t2.City ORDER BY COUNT(*) DESC LIMIT 1;"
+    },
+    {
+        "id": 15,
+        "status": "ERROR: Execution failed on sql 'SELECT T1.Title FROM Album AS T1 INNER JOIN InvoiceLine AS T2 ON T1.AlbumId = T2.AlbumId GROUP BY T1.AlbumId HAVING SUM(T2.UnitPrice * T2.Quantity) > 20;': no such column: T2.AlbumId"
+    }
+]

src/database/Chinook_Sqlite.sqlite

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7651ba378ac2fcd0dfc3c66fb101f7a7eed3ba39a612ec642b96e20702061f15
+size 1007616

src/database/db_manager.py

@@ -0,0 +1,48 @@
+# This module provides a function to establish a connection to the SQLite database used in the NL2SQL project. It also includes a test block to verify the connection and list the tables in the database.
+
+import sqlite3
+import os
+
+# Get the path to the database file
+DB_PATH = os.path.join(os.path.dirname(__file__), 'Chinook_Sqlite.sqlite')
+
+def get_db_connection():
+    """Establishes a connection to the SQLite database."""
+    try:
+        connection = sqlite3.connect(DB_PATH)
+        return connection
+    except sqlite3.Error as e:
+        print(f"Error connecting to database: {e}")
+        return None
+    
+# Test the database connection
+if __name__ == "__main__":
+    connection = get_db_connection()
+    if connection:
+        print("Database connection successful!")
+        cursor = connection.cursor()
+        cursor.execute("SELECT name FROM sqlite_master WHERE type='table';")
+        print("Tables in the database:", cursor.fetchall())
+        connection.close()
+    else:
+        print("Failed to connect to the database.")
+
+# Extract Schema Information for LLM Prompts
+def get_schema_context():
+    """Extracts the database schema information to be used in LLM prompts."""
+    connection = get_db_connection()
+    if not connection:
+        return "Unable to connect to the database to retrieve schema information."
+    
+    cursor = connection.cursor()
+    cursor.execute("SELECT name FROM sqlite_master WHERE type='table';")
+    tables = [t[0] for t in cursor.fetchall() if not t[0].startswith('sqlite_')]
+
+    schema_text = ""
+    for table in tables:
+        cursor.execute(f"PRAGMA table_info({table});")
+        columns = [f"{c[1]} ({c[2]})" for c in cursor.fetchall()]
+        schema_text += f"Table {table}: {', '.join(columns)}\n"
+    connection.close()
+    return schema_text
+    

src/nl2sql/qwen_engine.py

@@ -0,0 +1,63 @@
+# Inference Script for Qwen-2.5:1B
+
+import requests
+import json
+
+def query_qwen(prompt):
+    url = "http://localhost:11434/api/generate"
+    payload = {
+        "model": "qwen2.5-coder:1.5b",
+        "prompt": prompt,
+        "stream": False,
+        "options": {
+            # Set temperature to 0 for deterministic output
+            "temperature": 0
+        }
+    }
+    response = requests.post(url, json=payload)
+    raw_response = response.json().get("response", "").strip()
+
+    # Clean the response to extract only the SQL query
+    if raw_response.startswith("```sql") and raw_response.endswith("```"):
+        return raw_response.replace("```sql", "").replace("```", "").strip()
+    else:
+        return raw_response 
+
+def build_qwen_prompt(question, schema):
+    return f"""## Instructions:
+    You are an expert SQLite assistant that converts natural language questions into SQL queries.
+
+    Use ONLY SQLite syntax.
+
+    Important database relationships:
+    - Artist(ArtistId) -> Album(ArtistId)
+    - Album(AlbumId) -> Track(AlbumId)
+    - Track(GenreId) -> Genre(GenreId)
+    - Track(MediaTypeId) -> MediaType(MediaTypeId)
+    - Track(TrackId) -> InvoiceLine(TrackId)
+    - InvoiceLine(InvoiceId) -> Invoice(InvoiceId)
+    - Customer(CustomerId) -> Invoice(CustomerId)
+    - Customer(SupportRepId) -> Employee(EmployeeId)
+    - Playlist(PlaylistId) -> PlaylistTrack(PlaylistId)
+    - PlaylistTrack(TrackId) -> Track(TrackId)
+
+    SQL Rules:
+    1. Use JOIN when data comes from multiple tables.
+    2. Use GROUP BY when using COUNT, SUM, or AVG.
+    3. Use HAVING for filtering aggregated values.
+    4. Use ORDER BY for ranking results.
+    5. Use LIMIT instead of TOP.
+    6. Always produce executable SQLite SQL.
+    7. Always use table aliases (t, a, ar, g, etc.) when joining tables.
+
+    Return ONLY the SQL query.
+    Do not provide explanations.
+
+    ## Database Schema:
+    {schema}
+
+    ## Question:
+    {question}
+
+    ## SQL Query:
+    """

src/scripts/evaluate.py

@@ -0,0 +1,69 @@
+# Evaluation Script for NL2SQL Models
+
+import json
+import pandas as pd
+from src.database.db_manager import get_db_connection, get_schema_context
+from src.nl2sql.qwen_engine import query_qwen, build_qwen_prompt
+
+def compare_results(df_generated_sql, df_gold_sql):
+    """Compares the generated SQL query with the expected SQL query."""
+    if df_generated_sql is None or df_gold_sql is None:
+        return False
+    try:
+        return df_generated_sql.values.tolist() == df_gold_sql.values.tolist()
+    except Exception as e:
+        print(f"Error comparing results: {e}")
+        return False
+    
+def run_evaluation():
+    # Load test cases from a JSON file
+    with open('src/scripts/test_cases.json', 'r') as f:
+        test_cases = json.load(f)
+
+    results = []
+    correct_count = 0
+    schema = get_schema_context()
+
+    print(f"Running evaluation on {len(test_cases)} test cases...\n")
+
+    for case in test_cases:
+        print(f"Testing ID {case['id']}: {case['question'][:50]}...")
+        
+        # Generate SQL query using the LLM
+        prompt = build_qwen_prompt(case['question'], schema)
+        generated_sql = query_qwen(prompt)
+
+        # Execute both generated SQL and gold SQL
+        connection = get_db_connection()
+        try:
+            df_generated = pd.read_sql_query(generated_sql, connection)
+            df_gold = pd.read_sql_query(case['gold_sql'], connection)
+
+            # Compare results
+            is_correct = compare_results(df_generated, df_gold)
+            if is_correct:
+                correct_count += 1
+            
+            results.append({
+                "id": case['id'],
+                "status": "PASS" if is_correct else "FAIL",
+                "generated": generated_sql
+            })
+        except Exception as e:
+            results.append({"id": case['id'], "status": f"ERROR: {str(e)}"})
+        finally:
+            connection.close()
+    
+    # Print summary
+    accuracy = (correct_count / len(test_cases)) * 100
+    print(f"\n EVALUATION COMPLETE")
+    print(f"Total Test Cases: {len(test_cases)}")
+    print(f"Correctly Generated SQL: {correct_count}")
+    print(f"Execution Accuracy: {accuracy:.2f}%")
+
+    # Save results to a JSON file
+    with open('evaluation_results.json', 'w') as f:
+        json.dump(results, f, indent=4)
+
+if __name__ == "__main__":
+    run_evaluation()

src/scripts/test_cases.json

@@ -0,0 +1,77 @@
+[
+    {
+        "id": 1,
+        "question": "How many tracks are there in each genre? List the genre name and the count.",
+        "gold_sql": "SELECT t.Genre, COUNT(t.TrackId) AS TrackCount FROM Track t GROUP BY t.Genre;"
+    },
+    {
+        "id": 2,
+        "question": "Provide a list of all albums and the name of the artist who created them.",
+        "gold_sql": "SELECT a.Title, ar.Name FROM Album a JOIN Artist ar ON a.ArtistId = ar.ArtistId;"
+    },
+    {
+        "id": 3,
+        "question": "What is the total revenue generated from each country?",
+        "gold_sql": "SELECT BillingCountry, SUM(Total) FROM Invoice GROUP BY BillingCountry;"
+    },
+    {
+        "id": 4,
+        "question": "Show the full names of all employees who are Sales Support Agents.",
+        "gold_sql": "SELECT e.FirstName, e.LastName FROM Employee e WHERE e.Title = 'Sales Support Agent';"
+    },
+    {
+        "id": 5,
+        "question": "List the top 5 customers who have spent the most money.",
+        "gold_sql": "SELECT c.CustomerId, c.FirstName, c.LastName, SUM(i.Total) as TotalSpent FROM Customer c JOIN Invoice i ON c.CustomerId = i.CustomerId GROUP BY c.CustomerId, c.FirstName, c.LastName ORDER BY TotalSpent DESC LIMIT 5;"
+    },
+    {
+        "id": 6,
+        "question": "List all Rock songs and the artists who performed them.",
+        "gold_sql": "SELECT t.Name, ar.Name FROM Track t JOIN Genre g ON t.GenreId = g.GenreId JOIN Album a ON t.AlbumId = a.AlbumId JOIN Artist ar ON a.ArtistId = ar.ArtistId WHERE g.Name = 'Rock';"
+    },
+    {
+        "id": 7,
+        "question": "Find the total number of tracks sold for each media type.",
+        "gold_sql": "SELECT m.Name, COUNT(il.TrackId) FROM MediaType m JOIN Track t ON m.MediaTypeId = t.MediaTypeId JOIN InvoiceLine il ON t.TrackId = il.TrackId GROUP BY m.Name;"
+    },
+    {
+        "id": 8,
+        "question": "Show the names of all tracks that appear on the 'TV Shows' playlist.",
+        "gold_sql": "SELECT t.Name FROM Track t JOIN PlaylistTrack pt ON t.TrackId = pt.TrackId JOIN Playlist p ON pt.PlaylistId = p.PlaylistId WHERE p.Name = 'TV Shows';"
+    },
+    {
+        "id": 9,
+        "question": "Which artist has the most tracks? Give the name and count.",
+        "gold_sql": "SELECT ar.Name, COUNT(t.TrackId) FROM Artist ar JOIN Album a ON ar.ArtistId = a.ArtistId JOIN Track t ON a.AlbumId = t.AlbumId GROUP BY ar.Name ORDER BY COUNT(t.TrackId) DESC LIMIT 1;"
+    },
+    {
+        "id": 10,
+        "question": "Which genres have more than 100 tracks?",
+        "gold_sql": "SELECT g.Name, COUNT(t.TrackId) as TrackCount FROM Genre g JOIN Track t ON g.GenreId = t.GenreId GROUP BY g.Name HAVING TrackCount > 100;"
+    },
+    {
+        "id": 11,
+        "question": "Who is the best-selling artist by total revenue? Provide the artist's name and total revenue.",
+        "gold_sql": "SELECT ar.Name, SUM(i.Total) as TotalRevenue FROM Artist ar JOIN Album a ON ar.ArtistId = a.ArtistId JOIN Track t ON a.AlbumId = t.AlbumId JOIN InvoiceLine il ON t.TrackId = il.TrackId JOIN Invoice i ON il.InvoiceId = i.InvoiceId GROUP BY ar.Name ORDER BY TotalRevenue DESC LIMIT 1;"
+    },
+    {
+        "id": 12,
+        "question": "Find the average length of tracks in seconds for each album. List the album title and average length.",
+        "gold_sql": "SELECT a.Title, AVG(t.Milliseconds) as AverageLength FROM Album a JOIN Track t ON a.AlbumId = t.AlbumId GROUP BY a.Title;"
+    },
+    {
+        "id": 13,
+        "question": "List customers helped by the employee Jane Peacock. Provide the customer's full name and the employee's full name.",
+        "gold_sql": "SELECT c.FirstName AS CustomerFirstName, c.LastName AS CustomerLastName, e.FirstName AS EmployeeFirstName, e.LastName AS EmployeeLastName FROM Customer c JOIN Employee e ON c.SupportRepId = e.EmployeeId WHERE e.FirstName = 'Jane' AND e.LastName = 'Peacock';"
+    },
+    {
+        "id": 14,
+        "question": "Which city had the highest number of invoices in 2013?",
+        "gold_sql": "SELECT BillingCity, COUNT(InvoiceId) FROM Invoice WHERE InvoiceDate LIKE '2013%' GROUP BY BillingCity ORDER BY 2 DESC LIMIT 1;"
+    },
+    {
+        "id": 15,
+        "question": "List albums with a total price greater than 20 dollars.",
+        "gold_sql": "SELECT al.Title, SUM(t.UnitPrice) FROM Album al JOIN Track t ON al.AlbumId = t.AlbumId GROUP BY al.Title HAVING SUM(t.UnitPrice) > 20;"
+    }
+]

test_bench.py

@@ -0,0 +1,36 @@
+# Test the database connection and schema extraction
+
+import pandas as pd
+from src.database.db_manager import get_db_connection, get_schema_context
+from src.nl2sql.qwen_engine import query_qwen, build_qwen_prompt
+
+def run_test(question):
+    print(f"\n[USER QUESTION]: {question}")
+
+    # Step 1: Get the database schema context from db_manager
+    schema = get_schema_context()
+
+    # Step 2: Build the prompt and Generate SQL query
+    prompt = build_qwen_prompt(question, schema)
+    generated_sql = query_qwen(prompt)
+    print(f"\n[GENERATED SQL]:\n{generated_sql}")
+
+    # Step 3: Execute the generated SQL query against the database
+    try:
+        connection = get_db_connection()
+        # Use pandas to execute the SQL query and display results
+        result_df = pd.read_sql_query(generated_sql, connection)
+        connection.close()
+
+        print("[EXECUTION SUCCESS]:")
+        if result_df.empty:
+            print("Query returned no results.")
+        else:
+            print(result_df.head(5))  # Display the first 5 rows of the result
+    except Exception as e:
+        print(f"[EXECUTION ERROR]: {e}")
+
+if __name__ == "__main__":
+    # Example test question
+    test_question = "What is the total revenue generated from each country?"
+    run_test(test_question)