Update Agent Response using NL

app/main.py

@@ -2,8 +2,8 @@
 # Main entry point for the NL2SQL application
 import os
 from dotenv import load_dotenv
-from src.nl2sql.sql_agent import nl2sql_agent
-from src.scripts.evaluate_hf import run_evaluation
+from src.scripts.interactive_mode import run_interactiveMode
+from src.scripts.evaluation_mode import run_evaluation
 
 load_dotenv()
 # Load HuggingFace API token from environment variable
@@ -11,32 +11,6 @@ hf_token = os.getenv("HF_TOKEN")
 if not hf_token:
     raise ValueError("HuggingFace API token not found!")
 
-# User prompt question manually and see the agent's response
-def interactive_mode():
-    """Allows user to manually type questions and get agent's response."""
-    print("\n========= Interactive NL2SQL Mode =========")
-    print("Type 'exit' or 'q' to return to the main menu.\n")
-
-    while True:
-        question = input("\nEnter your question: ")
-        if question.lower() in ['exit', 'q']:
-            break
-        if not question.strip():
-            continue
-
-        print("\nProcessing your question...")
-        response = nl2sql_agent(question)
-
-        print("\n========= Agent Response =========")
-        print(f"Status: {response.get('status')}")
-        print(f"Generated SQL:\n{response.get('query')}")
-
-        if response.get('status') == 'success':
-            print(f"\nresults (First 5 rows):\n{response.get('results')[:5]}")
-        else:
-            print(f"\nError Details:\n{response.get('error')}")
-        print("==================================\n")
-
 def main():
     """Main application entry point and interactive menu"""
     while True:
@@ -51,7 +25,7 @@ def main():
         choice = input("Select an option (1-3): ")
 
         if choice == '1':
-            interactive_mode()
+            run_interactiveMode()
         elif choice == '2':
             run_evaluation()
         elif choice == '3':

src/nl2sql/sql_agent.py

@@ -40,6 +40,16 @@ Do not include any explanations, markdown formatting, or code blocks.
 
 Corrected SQL Query:"""
 
+# Generate text response
+NL_RESPONSE_TEMPLATE = """You are a helpful data assisstant.
+The user asked the following question: "{question}"
+The database returned the following results: {results}
+
+Provide a direct, natural language answer to the user's question using ONLY the provided data.
+Keep it brief. Do not explain the SQL query or mention the database schema.
+
+Answer:"""
+
 prompt_template = PromptTemplate(
     input_variables = ["schema", "question"],
     template = SQL_PROMPT_TEMPLATE
@@ -50,6 +60,11 @@ refinement_prompt = PromptTemplate(
     template = REFINEMENT_PROMPT_TEMPLATE
 )
 
+nl_response_template = PromptTemplate(
+    input_variables = ["question", "results"],
+    template = NL_RESPONSE_TEMPLATE
+)
+
 # Clean the output
 def clean_sql(raw_sql: str) -> str:
     """
@@ -83,6 +98,7 @@ def nl2sql_agent(user_question: str, max_retries: int = 3) -> dict:
     # LangChain Pipeline: Pipe prompt into LLM
     chain = prompt_template | llm
     refinement_chain = refinement_prompt | llm
+    nl_chain = nl_response_template | llm
 
     current_sql = ""
     error_message = ""
@@ -126,10 +142,18 @@ def nl2sql_agent(user_question: str, max_retries: int = 3) -> dict:
                 
                 if attempt > 1:
                     print(f"SQL query executed successfully after {attempt} attempts.")
+                
+                # Generate natural language response based on the results
+                print("Generating natural language response based on query results...")
+                nl_response = nl_chain.invoke({
+                    "question": user_question,
+                    "results": str(results)
+                })
 
                 return {
                     "query": generated_sql,
                     "results": results,
+                    "nl_response": nl_response,
                     "status": "success",
                     "attempts": attempt
                 }

src/scripts/evaluation_mode.py

@@ -0,0 +1,107 @@
+# Path: src/scripts/evaluation_mode.py
+# Evaluation script for Hugging Face SQL generation.
+import json
+from pathlib import Path
+import pandas as pd
+from src.database.db_manager import get_db_connection
+from src.nl2sql.sql_agent import nl2sql_agent
+
+TEST_CASES_PATH = Path("src/scripts/test_cases.json")
+RESULTS_PATH = Path("hf_evaluation_results.json")
+
+def _normalize_dataframe(dataframe: pd.DataFrame) -> pd.DataFrame:
+    # Normalize dataframe to ensure accurate comparison
+    normalized = dataframe.copy()
+    normalized.columns = [str(column).lower() for column in normalized.columns]
+
+    for column in normalized.columns:
+        normalized[column] = normalized[column].map(
+            lambda value: round(float(value), 6)
+            if isinstance(value, float)
+            else value
+        )
+
+    sort_columns = list(normalized.columns)
+    if sort_columns:
+        normalized = normalized.sort_values(by=sort_columns, kind="mergesort").reset_index(drop=True)
+
+    return normalized
+
+# Compare generated SQL results with expected results
+def compare_results(df_generated: pd.DataFrame, df_gold: pd.DataFrame) -> bool:
+    """Compare generated and expected query results."""
+    if df_generated is None or df_gold is None:
+        return False
+
+    try:
+        normalized_generated = _normalize_dataframe(df_generated)
+        normalized_gold = _normalize_dataframe(df_gold)
+        return normalized_generated.equals(normalized_gold)
+    except Exception as error:
+        print(f"Error comparing results: {error}")
+        return False
+
+def run_evaluation():
+    if not TEST_CASES_PATH.exists():
+        print(f"Error: Could not find test cases at {TEST_CASES_PATH}")
+        return
+    
+    with TEST_CASES_PATH.open("r", encoding="utf-8") as handle:
+        test_cases = json.load(handle)
+
+    results = []
+    correct_count = 0
+
+    print(f"Running evaluation on {len(test_cases)} test cases...\n")
+
+    for case in test_cases:
+        question = case["question"]
+        print(f"Testing ID {case['id']}: {question[:50]}...")
+
+        # Implement agent to handle RAG retrieval and SQL generation
+        agent_response = nl2sql_agent(user_question=question)
+        generated_sql = agent_response.get("query", "")
+
+        connection = get_db_connection()
+        if connection is None:
+            raise RuntimeError("Unable to connect to the SQLite database.")
+
+        try:
+            df_generated = pd.read_sql_query(generated_sql, connection)
+            df_gold = pd.read_sql_query(case["gold_sql"], connection)
+
+            is_correct = compare_results(df_generated, df_gold)
+            if is_correct:
+                correct_count += 1
+
+            results.append(
+                {
+                    "id": case["id"],
+                    "question": question,
+                    "status": "PASS" if is_correct else "FAIL",
+                    "generated_sql": generated_sql,
+                    "gold_sql": case["gold_sql"],
+                }
+            )
+        except Exception as error:
+            results.append(
+                {
+                    "id": case["id"],
+                    "question": question,
+                    "status": "ERROR",
+                    "generated_sql": generated_sql,
+                    "gold_sql": case["gold_sql"],
+                    "error": str(error),
+                }
+            )
+        finally:
+            connection.close()
+
+    accuracy = (correct_count / len(test_cases)) * 100 if test_cases else 0.0
+    print("\nEVALUATION COMPLETE")
+    print(f"Total Test Cases: {len(test_cases)}")
+    print(f"Correctly Generated SQL: {correct_count} / {len(test_cases)}")
+    print(f"Execution Accuracy: {accuracy:.2f}%")
+
+    with RESULTS_PATH.open("w", encoding="utf-8") as handle:
+        json.dump(results, handle, indent=4)

src/scripts/interactive_mode.py

@@ -0,0 +1,28 @@
+# Path: src/scripts/interactive_mode.py
+# Interactive mode: Allows user to manually type questions and see the agent's response
+from src.nl2sql.sql_agent import nl2sql_agent
+
+def run_interactiveMode():
+    """ Allows user to manually type questions and get agent's response."""
+    print("\n========= Interactive NL2SQL Mode =========")
+    print("Type 'exit' or 'q' to return to the main menu.\n")
+
+    while True:
+        question = input("\nEnter your question: ")
+        if question.lower() in ['exit', 'q']:
+            break
+        if not question.strip():
+            continue
+
+        print("\nProcessing your question...")
+        response = nl2sql_agent(question)
+
+        print("\n========= Agent Response =========")
+        if response.get('status') == 'success':
+            print(f"Answer: {response.get('nl_response')}\n")
+            print(f"Generated SQL:\n{response.get('query')}\n")
+        else:
+            print(f"Status: {response.get('status')}")
+            print(f"Generated SQL:\n{response.get('query')}")
+            print(f"\nError Details:\n{response.get('error')}")
+        print("==================================\n")