Delete unused files

evaluation_results.json

@@ -1,72 +0,0 @@
-[
-    {
-        "id": 1,
-        "status": "ERROR: Execution failed on sql 'SELECT t.Genre, COUNT(t.TrackId) AS TrackCount FROM Track t GROUP BY t.Genre;': no such column: t.Genre"
-    },
-    {
-        "id": 2,
-        "status": "PASS",
-        "generated": "SELECT t1.Title, ar.Name FROM Album AS t1 INNER JOIN Artist AS ar ON t1.ArtistId = ar.ArtistId;"
-    },
-    {
-        "id": 3,
-        "status": "ERROR: Execution failed on sql 'SELECT T4.Country, SUM(T3.UnitPrice * T3.Quantity) as TotalRevenue\nFROM InvoiceLine AS T1\nJOIN Track AS T2 ON T1.TrackId = T2.TrackId\nJOIN Album AS T3 ON T2.AlbumId = T3.AlbumId\nJOIN Genre AS T4 ON T2.GenreId = T4.GenreId\nGROUP BY T4.Country;': no such column: T4.Country"
-    },
-    {
-        "id": 4,
-        "status": "ERROR: Execution failed on sql 'SELECT e.FirstName, e.LastName FROM Employee AS e JOIN Customer AS c ON e.SupportRepId = c.CustomerId WHERE c.Company LIKE 'Sales Support';': no such column: e.SupportRepId"
-    },
-    {
-        "id": 5,
-        "status": "FAIL",
-        "generated": "SELECT c.FirstName, c.LastName, SUM(il.UnitPrice * il.Quantity) as TotalSpent\nFROM Customer c\nJOIN Invoice i ON c.CustomerId = i.CustomerId\nJOIN InvoiceLine il ON i.InvoiceId = il.InvoiceId\nGROUP BY c.CustomerId\nORDER BY TotalSpent DESC\nLIMIT 5;"
-    },
-    {
-        "id": 6,
-        "status": "FAIL",
-        "generated": "SELECT T3.Name AS ArtistName, T1.Name AS SongName\nFROM Track AS T1\nJOIN Album AS T2 ON T1.AlbumId = T2.AlbumId\nJOIN Genre AS T3 ON T1.GenreId = T3.GenreId\nWHERE T3.Name = 'Rock'\nGROUP BY T1.TrackId;"
-    },
-    {
-        "id": 7,
-        "status": "ERROR: Execution failed on sql 'SELECT T3.Name AS MediaTypeName, COUNT(T1.TrackId) AS TotalTracksSold\nFROM Track AS T1\nJOIN MediaType AS T2 ON T1.MediaTypeId = T2.MediaTypeId\nGROUP BY T2.Name;': no such column: T3.Name"
-    },
-    {
-        "id": 8,
-        "status": "PASS",
-        "generated": "SELECT T3.Name FROM Playlist AS T1 INNER JOIN PlaylistTrack AS T2 ON T1.PlaylistId = T2.PlaylistId INNER JOIN Track AS T3 ON T2.TrackId = T3.TrackId WHERE T1.Name = 'TV Shows';"
-    },
-    {
-        "id": 9,
-        "status": "PASS",
-        "generated": "SELECT a.Name, COUNT(t.TrackId) AS TrackCount\nFROM Artist a\nJOIN Album ar ON a.ArtistId = ar.ArtistId\nJOIN Track t ON ar.AlbumId = t.AlbumId\nGROUP BY a.ArtistId\nORDER BY TrackCount DESC\nLIMIT 1;"
-    },
-    {
-        "id": 10,
-        "status": "FAIL",
-        "generated": "SELECT g.Name FROM Genre g JOIN Track t ON g.GenreId = t.GenreId GROUP BY g.Name HAVING COUNT(t.TrackId) > 100;"
-    },
-    {
-        "id": 11,
-        "status": "FAIL",
-        "generated": "SELECT a.Name, SUM(il.UnitPrice * il.Quantity) as TotalRevenue\nFROM Artist a\nJOIN Album al ON a.ArtistId = al.ArtistId\nJOIN Track tr ON al.AlbumId = tr.AlbumId\nJOIN InvoiceLine il ON tr.TrackId = il.TrackId\nGROUP BY a.ArtistId\nORDER BY TotalRevenue DESC\nLIMIT 1;"
-    },
-    {
-        "id": 12,
-        "status": "FAIL",
-        "generated": "SELECT a.Title, AVG(t.Milliseconds) / 1000 AS AverageLengthInSeconds\nFROM Album a\nJOIN Track t ON a.AlbumId = t.AlbumId\nGROUP BY a.AlbumId;"
-    },
-    {
-        "id": 13,
-        "status": "PASS",
-        "generated": "SELECT c.FirstName, c.LastName, e.FirstName, e.LastName\nFROM Customer c\nJOIN Employee e ON c.SupportRepId = e.EmployeeId\nWHERE e.LastName = 'Peacock';"
-    },
-    {
-        "id": 14,
-        "status": "PASS",
-        "generated": "SELECT t2.City FROM Invoice AS t1 INNER JOIN Customer AS t2 ON t1.CustomerId = t2.CustomerId WHERE strftime('%Y', t1.InvoiceDate) = '2013' GROUP BY t2.City ORDER BY COUNT(*) DESC LIMIT 1;"
-    },
-    {
-        "id": 15,
-        "status": "ERROR: Execution failed on sql 'SELECT T1.Title FROM Album AS T1 INNER JOIN InvoiceLine AS T2 ON T1.AlbumId = T2.AlbumId GROUP BY T1.AlbumId HAVING SUM(T2.UnitPrice * T2.Quantity) > 20;': no such column: T2.AlbumId"
-    }
-]

src/nl2sql/qwen_engine.py

@@ -1,63 +0,0 @@
-# Inference Script for Qwen-2.5:1B
-
-import requests
-import json
-
-def query_qwen(prompt):
-    url = "http://localhost:11434/api/generate"
-    payload = {
-        "model": "qwen2.5-coder:1.5b",
-        "prompt": prompt,
-        "stream": False,
-        "options": {
-            # Set temperature to 0 for deterministic output
-            "temperature": 0
-        }
-    }
-    response = requests.post(url, json=payload)
-    raw_response = response.json().get("response", "").strip()
-
-    # Clean the response to extract only the SQL query
-    if raw_response.startswith("```sql") and raw_response.endswith("```"):
-        return raw_response.replace("```sql", "").replace("```", "").strip()
-    else:
-        return raw_response 
-
-def build_qwen_prompt(question, schema):
-    return f"""## Instructions:
-    You are an expert SQLite assistant that converts natural language questions into SQL queries.
-
-    Use ONLY SQLite syntax.
-
-    Important database relationships:
-    - Artist(ArtistId) -> Album(ArtistId)
-    - Album(AlbumId) -> Track(AlbumId)
-    - Track(GenreId) -> Genre(GenreId)
-    - Track(MediaTypeId) -> MediaType(MediaTypeId)
-    - Track(TrackId) -> InvoiceLine(TrackId)
-    - InvoiceLine(InvoiceId) -> Invoice(InvoiceId)
-    - Customer(CustomerId) -> Invoice(CustomerId)
-    - Customer(SupportRepId) -> Employee(EmployeeId)
-    - Playlist(PlaylistId) -> PlaylistTrack(PlaylistId)
-    - PlaylistTrack(TrackId) -> Track(TrackId)
-
-    SQL Rules:
-    1. Use JOIN when data comes from multiple tables.
-    2. Use GROUP BY when using COUNT, SUM, or AVG.
-    3. Use HAVING for filtering aggregated values.
-    4. Use ORDER BY for ranking results.
-    5. Use LIMIT instead of TOP.
-    6. Always produce executable SQLite SQL.
-    7. Always use table aliases (t, a, ar, g, etc.) when joining tables.
-
-    Return ONLY the SQL query.
-    Do not provide explanations.
-
-    ## Database Schema:
-    {schema}
-
-    ## Question:
-    {question}
-
-    ## SQL Query:
-    """

src/scripts/evaluate.py

@@ -1,69 +0,0 @@
-# Evaluation Script for NL2SQL Models
-
-import json
-import pandas as pd
-from src.database.db_manager import get_db_connection, get_schema_context
-from src.nl2sql.qwen_engine import query_qwen, build_qwen_prompt
-
-def compare_results(df_generated_sql, df_gold_sql):
-    """Compares the generated SQL query with the expected SQL query."""
-    if df_generated_sql is None or df_gold_sql is None:
-        return False
-    try:
-        return df_generated_sql.values.tolist() == df_gold_sql.values.tolist()
-    except Exception as e:
-        print(f"Error comparing results: {e}")
-        return False
-    
-def run_evaluation():
-    # Load test cases from a JSON file
-    with open('src/scripts/test_cases.json', 'r') as f:
-        test_cases = json.load(f)
-
-    results = []
-    correct_count = 0
-    schema = get_schema_context()
-
-    print(f"Running evaluation on {len(test_cases)} test cases...\n")
-
-    for case in test_cases:
-        print(f"Testing ID {case['id']}: {case['question'][:50]}...")
-        
-        # Generate SQL query using the LLM
-        prompt = build_qwen_prompt(case['question'], schema)
-        generated_sql = query_qwen(prompt)
-
-        # Execute both generated SQL and gold SQL
-        connection = get_db_connection()
-        try:
-            df_generated = pd.read_sql_query(generated_sql, connection)
-            df_gold = pd.read_sql_query(case['gold_sql'], connection)
-
-            # Compare results
-            is_correct = compare_results(df_generated, df_gold)
-            if is_correct:
-                correct_count += 1
-            
-            results.append({
-                "id": case['id'],
-                "status": "PASS" if is_correct else "FAIL",
-                "generated": generated_sql
-            })
-        except Exception as e:
-            results.append({"id": case['id'], "status": f"ERROR: {str(e)}"})
-        finally:
-            connection.close()
-    
-    # Print summary
-    accuracy = (correct_count / len(test_cases)) * 100
-    print(f"\n EVALUATION COMPLETE")
-    print(f"Total Test Cases: {len(test_cases)}")
-    print(f"Correctly Generated SQL: {correct_count}")
-    print(f"Execution Accuracy: {accuracy:.2f}%")
-
-    # Save results to a JSON file
-    with open('evaluation_results.json', 'w') as f:
-        json.dump(results, f, indent=4)
-
-if __name__ == "__main__":
-    run_evaluation()

src/scripts/evaluate_hf.py

@@ -1,108 +0,0 @@
-# src/scripts/evaluate_hf.py
-# Evaluation script for Hugging Face SQL generation.
-
-import json
-from pathlib import Path
-import pandas as pd
-from src.database.db_manager import get_db_connection
-from src.nl2sql.sql_agent import nl2sql_agent
-
-TEST_CASES_PATH = Path("src/scripts/test_cases.json")
-RESULTS_PATH = Path("hf_evaluation_results.json")
-
-def _normalize_dataframe(dataframe: pd.DataFrame) -> pd.DataFrame:
-    # Normalize dataframe to ensure accurate comparison
-    normalized = dataframe.copy()
-    normalized.columns = [str(column).lower() for column in normalized.columns]
-
-    for column in normalized.columns:
-        normalized[column] = normalized[column].map(
-            lambda value: round(float(value), 6)
-            if isinstance(value, float)
-            else value
-        )
-
-    sort_columns = list(normalized.columns)
-    if sort_columns:
-        normalized = normalized.sort_values(by=sort_columns, kind="mergesort").reset_index(drop=True)
-
-    return normalized
-
-# Compare generated SQL results with expected results
-def compare_results(df_generated: pd.DataFrame, df_gold: pd.DataFrame) -> bool:
-    """Compare generated and expected query results."""
-    if df_generated is None or df_gold is None:
-        return False
-
-    try:
-        normalized_generated = _normalize_dataframe(df_generated)
-        normalized_gold = _normalize_dataframe(df_gold)
-        return normalized_generated.equals(normalized_gold)
-    except Exception as error:
-        print(f"Error comparing results: {error}")
-        return False
-
-def run_evaluation():
-    if not TEST_CASES_PATH.exists():
-        print(f"Error: Could not find test cases at {TEST_CASES_PATH}")
-        return
-    
-    with TEST_CASES_PATH.open("r", encoding="utf-8") as handle:
-        test_cases = json.load(handle)
-
-    results = []
-    correct_count = 0
-
-    print(f"Running evaluation on {len(test_cases)} test cases...\n")
-
-    for case in test_cases:
-        question = case["question"]
-        print(f"Testing ID {case['id']}: {question[:50]}...")
-
-        # Implement agent to handle RAG retrieval and SQL generation
-        agent_response = nl2sql_agent(user_question=question)
-        generated_sql = agent_response.get("query", "")
-
-        connection = get_db_connection()
-        if connection is None:
-            raise RuntimeError("Unable to connect to the SQLite database.")
-
-        try:
-            df_generated = pd.read_sql_query(generated_sql, connection)
-            df_gold = pd.read_sql_query(case["gold_sql"], connection)
-
-            is_correct = compare_results(df_generated, df_gold)
-            if is_correct:
-                correct_count += 1
-
-            results.append(
-                {
-                    "id": case["id"],
-                    "question": question,
-                    "status": "PASS" if is_correct else "FAIL",
-                    "generated_sql": generated_sql,
-                    "gold_sql": case["gold_sql"],
-                }
-            )
-        except Exception as error:
-            results.append(
-                {
-                    "id": case["id"],
-                    "question": question,
-                    "status": "ERROR",
-                    "generated_sql": generated_sql,
-                    "gold_sql": case["gold_sql"],
-                    "error": str(error),
-                }
-            )
-        finally:
-            connection.close()
-
-    accuracy = (correct_count / len(test_cases)) * 100 if test_cases else 0.0
-    print("\nEVALUATION COMPLETE")
-    print(f"Total Test Cases: {len(test_cases)}")
-    print(f"Correctly Generated SQL: {correct_count} / {len(test_cases)}")
-    print(f"Execution Accuracy: {accuracy:.2f}%")
-
-    with RESULTS_PATH.open("w", encoding="utf-8") as handle:
-        json.dump(results, handle, indent=4)

test_bench.py

@@ -1,36 +0,0 @@
-# Test the database connection and schema extraction
-
-import pandas as pd
-from src.database.db_manager import get_db_connection, get_schema_context
-from src.nl2sql.qwen_engine import query_qwen, build_qwen_prompt
-
-def run_test(question):
-    print(f"\n[USER QUESTION]: {question}")
-
-    # Step 1: Get the database schema context from db_manager
-    schema = get_schema_context()
-
-    # Step 2: Build the prompt and Generate SQL query
-    prompt = build_qwen_prompt(question, schema)
-    generated_sql = query_qwen(prompt)
-    print(f"\n[GENERATED SQL]:\n{generated_sql}")
-
-    # Step 3: Execute the generated SQL query against the database
-    try:
-        connection = get_db_connection()
-        # Use pandas to execute the SQL query and display results
-        result_df = pd.read_sql_query(generated_sql, connection)
-        connection.close()
-
-        print("[EXECUTION SUCCESS]:")
-        if result_df.empty:
-            print("Query returned no results.")
-        else:
-            print(result_df.head(5))  # Display the first 5 rows of the result
-    except Exception as e:
-        print(f"[EXECUTION ERROR]: {e}")
-
-if __name__ == "__main__":
-    # Example test question
-    test_question = "What is the total revenue generated from each country?"
-    run_test(test_question)