Huggingface Deployment

.dockerignore

@@ -0,0 +1,87 @@
+# Exclude heavy folders from Docker image
+**/__pycache__
+**/.venv
+**/.git
+**/.ipynb_checkpoints
+**/hf_cache
+**/node_modules
+
+# Exclude local database data
+**/postgres-data
+**/mysql-data
+
+# Exclude large model weights and research artifacts
+Agent/Gemma3_1B_weights
+Agent/merged_gemma1b_text2Sql
+Agent/merged_gemma3_1b_text2Sql
+Agent/Qwen2.5_3b_weights
+Agent/Qwen2.5-3B-Instruct
+merged_qwen2_5_3b_text2sql
+merged_gemma3_1b_text2sql
+.venv-merge/
+
+# Exclude all Jupyter Notebooks EXCEPT the proof-of-work notebook
+**/*.ipynb
+!Fine_Tuning_Qwen_2_5_3B.ipynb
+**/.ipynb_checkpoints
+
+# Exclude redundant project clutter
+backups/
+debug_files/
+test_files/
+test_results/
+dump-mysql.bat
+dump-postgres.bat
+agentic_improvements.md
+invariants_analysis.md
+progress_summary.txt
+mcp_server.log
+merge.log
+response.json
+run-dev.ps1
+setup_ollama.bat
+
+
+# Exclude experimental/redundant Python scripts
+Agent/trial*.py
+Agent/test*.py
+Agent/old_*.py
+Agent/OLLAMA_GPU_NOTES.md
+**/debug_*.py
+
+# Exclude local uploads and generated files (keep empty folder)
+chat_store/docs/*
+!uploads/.gitkeep
+
+# Exclude local configuration (Hugging Face uses Secrets instead)
+.env
+.env.local
+
+# Exclude local logs and IDE settings
+*.log
+.vscode/
+.idea/
+.DS_Store
+
+# Exclude Python build artifacts
+__pycache__/
+*.pyc
+*.pyo
+*.pyd
+.Python
+env/
+venv/
+.venv/
+pip-log.txt
+pip-delete-this-directory.txt
+.toast
+.mypy_cache/
+.pytest_cache/
+.hypothesis/
+
+# Exclude large local data
+uploads/*
+!uploads/.gitkeep
+chat_store/docs/*
+!chat_store/docs/.gitkeep
+.idea/

.gitignore

@@ -0,0 +1,64 @@
+# Local Models and Weights
+Agent/*_weights/
+Agent/Qwen*/
+Agent/Gemma*/
+Agent/merged_*/
+merged_*/
+*.bin
+*.gguf
+*.onnx
+
+# Large Sample Database (Not needed in cloud)
+employee-sample-database/
+
+# Data and History
+chat_store/
+Agent/chat_store_private.json
+
+# Docker and DB
+docker/data/
+
+# Python
+__pycache__/
+*.py[cod]
+*.egg-info/
+*.egg
+*.pyo
+*.pyd
+*.so
+
+# Virtual environments
+venv/
+.venv/
+.env
+.env.local
+!.env.example
+
+# Jupyter Notebooks and checkpoints
+*.ipynb
+.ipynb_checkpoints/
+
+# Experimental/Redundant Scripts
+Agent/trial*.py
+Agent/old_*.py
+Agent/requirements-merge.txt
+debug_files/
+test_results/
+
+# Data and Uploads (Keep the folders, ignore the contents)
+uploads/*
+!uploads/.gitkeep
+chat_store/docs/*
+!chat_store/docs/.gitkeep
+
+# OS-specific and IDEs
+.DS_Store
+Thumbs.db
+.vscode/
+.idea/
+
+# Logs
+*.log
+mcp_server.log
+merge.log
+response.json

Agent/invariants.py

@@ -0,0 +1,485 @@
+"""Invariant classification and enforcement for text2sql without hardcoded SQL templates.
+
+All regeneration remains purely LLM-driven. We only express semantic/structural
+constraints ("invariants") and ask the model to satisfy missing ones.
+
+Categories:
+  MANAGER_LIST          -> list current department managers
+  DEPARTMENT_LIST       -> list department names
+  AVG_SALARY_BY_DEPT    -> average current salary per department
+  TITLE_COUNT           -> count employees with a given title (e.g. Senior Engineer)
+  GENERIC               -> anything else (no special invariants)
+
+Each invariant is represented as a dict:
+  {
+    'name': short identifier,
+    'description': human-readable guidance,
+    'check': callable(sql:str)->bool
+  }
+
+We DO NOT inject SQL fragments. We only:
+  - detect which invariants fail
+  - compile a focused regeneration prompt listing just the missing invariants
+  - request a corrected SQL (single statement ending with semicolon + LIMIT 100)
+
+This preserves the "no hardcoded fallback query" policy.
+"""
+from __future__ import annotations
+import os
+import re
+from typing import List, Dict, Callable, Optional
+
+CATEGORY_MANAGER_LIST = "MANAGER_LIST"
+CATEGORY_DEPT_LIST = "DEPARTMENT_LIST"
+CATEGORY_AVG_SALARY = "AVG_SALARY_BY_DEPT"
+CATEGORY_TITLE_COUNT = "TITLE_COUNT"
+CATEGORY_EMP_COUNT_BY_DEPT = "EMP_COUNT_BY_DEPT"
+CATEGORY_GENERIC = "GENERIC"
+
+# Newly added categories (extended taxonomy)
+CATEGORY_SIMPLE_COUNT_EMP = "SIMPLE_COUNT_EMP"
+CATEGORY_SIMPLE_COUNT_EMP_CURRENT = "SIMPLE_COUNT_EMP_CURRENT"
+CATEGORY_SIMPLE_COUNT_DEPT = "SIMPLE_COUNT_DEPT"
+CATEGORY_SALARY_EXTREMES_GLOBAL = "SALARY_EXTREMES_GLOBAL"
+CATEGORY_SALARY_RANGE_BY_DEPT = "SALARY_RANGE_BY_DEPT"
+CATEGORY_TOP_N_SALARIES = "TOP_N_SALARIES"
+CATEGORY_DEPT_MANAGER_LIST = "DEPT_MANAGER_LIST"  # dept + manager names
+CATEGORY_MANAGER_TENURE = "MANAGER_TENURE"
+CATEGORY_MANAGER_CHANGE_FREQUENCY = "MANAGER_CHANGE_FREQUENCY"
+CATEGORY_TITLE_DISTINCT_COUNT = "TITLE_DISTINCT_COUNT"
+CATEGORY_TITLE_SPECIFIC_COUNT = "TITLE_SPECIFIC_COUNT"
+CATEGORY_TITLE_DISTRIBUTION_BY_DEPT = "TITLE_DISTRIBUTION_BY_DEPT"
+CATEGORY_PROMOTION_PATH_ENGINEER_TO_SENIOR = "PROMOTION_PATH_ENGINEER_TO_SENIOR"
+CATEGORY_EMP_TENURE_CURRENT_DEPT = "EMP_TENURE_CURRENT_DEPT"
+CATEGORY_HIRING_TREND_BY_YEAR = "HIRING_TREND_BY_YEAR"
+CATEGORY_SALARY_TREND_BY_YEAR = "SALARY_TREND_BY_YEAR"
+CATEGORY_EMP_DEPT_CHANGES = "EMP_DEPT_CHANGES"
+CATEGORY_GENDER_DIST_BY_DEPT = "GENDER_DIST_BY_DEPT"
+CATEGORY_GENDER_PAY_GAP_BY_DEPT = "GENDER_PAY_GAP_BY_DEPT"
+CATEGORY_FEMALE_MANAGERS_COUNT = "FEMALE_MANAGERS_COUNT"
+CATEGORY_AVG_SALARY_BY_TITLE_AND_DEPT = "AVG_SALARY_BY_TITLE_AND_DEPT"
+CATEGORY_GENDER_MANAGEMENT_COUNT = "GENDER_MANAGEMENT_COUNT"
+CATEGORY_SENIOR_STAFF_SUPERLATIVE = "SENIOR_STAFF_SUPERLATIVE"
+
+Category = str
+
+# ---------------- Classification -----------------
+
+def classify_question(question: str) -> Category:
+    q = question.lower()
+    # Specific order matters: more specialized patterns first
+    if any(k in q for k in ["top", "highest-paid", "highest paid"]) and "salary" in q:
+        return CATEGORY_TOP_N_SALARIES
+    if any(kw in q for kw in ["total number of employees", "total employees"]) or ("count" in q and "employee" in q and "department" not in q and "current" not in q):
+        return CATEGORY_SIMPLE_COUNT_EMP
+    if ("how many" in q or "count" in q) and "currently" in q and "employee" in q:
+        return CATEGORY_SIMPLE_COUNT_EMP_CURRENT
+    if ("how many" in q or "number of" in q or "count" in q) and "departments" in q and "manager" not in q:
+        return CATEGORY_SIMPLE_COUNT_DEPT
+    if any(w in q for w in ["highest", "lowest"]) and "salary" in q and "department" not in q:
+        return CATEGORY_SALARY_EXTREMES_GLOBAL
+    if "salary range" in q and "department" in q:
+        return CATEGORY_SALARY_RANGE_BY_DEPT
+    if "most employees" in q and "department" in q:
+        return CATEGORY_EMP_COUNT_BY_DEPT
+    if "highest average salary" in q and "department" in q:
+        return CATEGORY_AVG_SALARY
+    if any(k in q for k in ["manager", "managers"]) and "department" in q:
+        if "tenure" in q or "long" in q or "position" in q:
+            return CATEGORY_MANAGER_TENURE
+        if "changed" in q or "changes" in q or "frequently" in q:
+            return CATEGORY_MANAGER_CHANGE_FREQUENCY
+        if "list" in q and "manager" in q and "department" in q:
+            return CATEGORY_DEPT_MANAGER_LIST
+        return CATEGORY_MANAGER_LIST
+    if ("how many" in q or "count" in q) and "job title" in q and ("exist" in q or "different" in q):
+        return CATEGORY_TITLE_DISTINCT_COUNT
+    if ("how many" in q or "count" in q) and any(t in q for t in ["senior engineer", "senior engineers"]):
+        return CATEGORY_TITLE_SPECIFIC_COUNT
+    if "distribution" in q and "title" in q and "department" in q:
+        return CATEGORY_TITLE_DISTRIBUTION_BY_DEPT
+    if "promoted" in q and "engineer" in q and "senior engineer" in q:
+        return CATEGORY_PROMOTION_PATH_ENGINEER_TO_SENIOR
+    if ("how long" in q or "tenure" in q) and "current" in q and "department" in q and "employee" in q:
+        return CATEGORY_EMP_TENURE_CURRENT_DEPT
+    if ("hiring trend" in q) or ("hire" in q and "year" in q):
+        return CATEGORY_HIRING_TREND_BY_YEAR
+    if ("salaries" in q or "salary" in q) and "changed" in q and ("over time" in q or "year" in q):
+        return CATEGORY_SALARY_TREND_BY_YEAR
+    if "changed departments" in q or ("employees" in q and "changed" in q and "department" in q):
+        return CATEGORY_EMP_DEPT_CHANGES
+    if "gender distribution" in q and "department" in q:
+        return CATEGORY_GENDER_DIST_BY_DEPT
+    if "gender pay gap" in q and "department" in q:
+        return CATEGORY_GENDER_PAY_GAP_BY_DEPT
+    if ("female" in q and "manager" in q and ("how many" in q or "count" in q)):
+        return CATEGORY_FEMALE_MANAGERS_COUNT
+    if "average salary" in q and "title" in q and "department" in q:
+        return CATEGORY_AVG_SALARY_BY_TITLE_AND_DEPT
+    if ("gender" in q and "management" in q) or ("each gender" in q and "management" in q):
+        return CATEGORY_GENDER_MANAGEMENT_COUNT
+    if ("most" in q and "senior" in q and "staff" in q) or ("senior staff" in q and "most" in q):
+        return CATEGORY_SENIOR_STAFF_SUPERLATIVE
+    if any(k in q for k in ["list departments", "department names", "department name", "departments list"]):
+        return CATEGORY_DEPT_LIST
+    if "average" in q and "salary" in q and "department" in q:
+        return CATEGORY_AVG_SALARY
+    if ("count" in q or "how many" in q or "number of" in q) and any(t in q for t in ["engineer", "engineers", "title"]):
+        return CATEGORY_TITLE_COUNT
+    if ("count" in q or "how many" in q or "number of" in q or any(s in q for s in ["most", "largest", "highest", "fewest", "least"])) and "department" in q and ("employee" in q or "employees" in q):
+        return CATEGORY_EMP_COUNT_BY_DEPT
+    return CATEGORY_GENERIC
+
+# --------------- Invariant Builders ---------------
+
+def _regex(pattern: str) -> Callable[[str], bool]:
+    rx = re.compile(pattern, re.IGNORECASE)
+    return lambda sql: bool(rx.search(sql))
+
+def _contains_all(*subs: str) -> Callable[[str], bool]:
+    def check(sql: str) -> bool:
+        l = sql.lower()
+        return all(s.lower() in l for s in subs)
+    return check
+
+def build_invariants(category: Category, question: str) -> List[Dict[str, any]]:
+    inv: List[Dict[str, any]] = []
+    if category == CATEGORY_MANAGER_LIST:
+        inv += [
+            {"name": "has_department_join", "description": "Join department (alias d) with dept_manager (alias dm)", "check": lambda s: re.search(r"join\s+dept_manager", s, re.IGNORECASE) and re.search(r"from\s+department", s, re.IGNORECASE)},
+            {"name": "has_employee_join", "description": "Join employee (alias e) for manager names", "check": _regex(r"join\s+employee")},
+            {"name": "selects_names", "description": "Select employee first_name and last_name", "check": lambda s: all(x in s.lower() for x in ["first_name", "last_name"])},
+            {"name": "filters_current", "description": "Filter current managers dm.to_date='9999-01-01'", "check": _regex(r"dm\.to_date\s*=\s*'9999-01-01'")},
+            {"name": "limit_100", "description": "Ends with LIMIT 100 semicolon", "check": _regex(r"limit\s+100;?\s*$")},
+        ]
+    elif category == CATEGORY_DEPT_LIST:
+        inv += [
+            {"name": "from_department", "description": "Query FROM department table", "check": _regex(r"from\s+department")},
+            {"name": "select_dept_name", "description": "Select dept_name column", "check": _regex(r"dept_name")},
+            {"name": "limit_100", "description": "Ends with LIMIT 100", "check": _regex(r"limit\s+100;?\s*$")},
+        ]
+    elif category == CATEGORY_AVG_SALARY:
+        inv += [
+            {"name": "joins_salary", "description": "Join salary s (FROM salary s)", "check": _regex(r"from\s+salary\s+s")},
+            {"name": "joins_dept_emp", "description": "Join dept_emp de", "check": _regex(r"join\s+dept_emp")},
+            {"name": "joins_department", "description": "Join department d", "check": _regex(r"join\s+department")},
+            {"name": "avg_salary", "description": "Uses AVG(s.salary)", "check": _regex(r"avg\s*\(\s*s\.salary\s*\)")},
+            {"name": "current_filters", "description": "Filters current rows s.to_date='9999-01-01' and de.to_date='9999-01-01'", "check": lambda s: "s.to_date='9999-01-01'" in s.lower() and "de.to_date='9999-01-01'" in s.lower()},
+            {"name": "group_by_dept", "description": "GROUP BY department name", "check": _regex(r"group\s+by\s+.*dept_name")},
+            {"name": "limit_100", "description": "Ends with LIMIT 100", "check": _regex(r"limit\s+100;?\s*$")},
+        ]
+    elif category == CATEGORY_TITLE_COUNT:
+        # Attempt to infer canonical title (not used in invariants; regeneration prompt may hint)
+        inv += [
+            {"name": "from_title", "description": "FROM title table (alias t)", "check": _regex(r"from\s+title")},
+            {"name": "alias_t", "description": "Alias 't' used for title table", "check": _regex(r"from\s+title\s+(?:as\s+)?t")},
+            {"name": "current_rows", "description": "Filter current rows t.to_date='9999-01-01'", "check": _regex(r"t\.to_date\s*=\s*'9999-01-01'")},
+            {"name": "count_distinct", "description": "COUNT(DISTINCT t.emp_no)", "check": _regex(r"count\s*\(\s*distinct\s+t\.emp_no\s*\)")},
+            {"name": "has_title_filter", "description": "Filter on t.title = '<Some Title>'", "check": _regex(r"t\.title\s*=\s*'[^']+'")} ,
+            {"name": "limit_100", "description": "Ends with LIMIT 100", "check": _regex(r"limit\s+100;?\s*$")},
+        ]
+    elif category == CATEGORY_EMP_COUNT_BY_DEPT:
+        inv += [
+            {"name": "has_dept_emp_join", "description": "Join dept_emp de with department d", "check": lambda s: re.search(r"join\s+dept_emp", s, re.IGNORECASE) and re.search(r"join\s+department|from\s+department", s, re.IGNORECASE)},
+            {"name": "select_dept_name", "description": "Select d.dept_name", "check": _regex(r"d\.dept_name|dept_name")},
+            {"name": "select_count", "description": "Include COUNT(*) or COUNT(de.emp_no)", "check": _regex(r"count\s*\(")},
+            {"name": "group_by", "description": "GROUP BY d.dept_name present", "check": _regex(r"group\s+by\s+.*dept_name")},
+            {"name": "limit_100", "description": "Ends with LIMIT 100", "check": _regex(r"limit\s+100;?\s*$")},
+            {"name": "count_alias", "description": "Provide an explicit alias e.g. COUNT(*) AS employee_count", "check": _regex(r"count\s*\(.*\)\s+as\s+employee_count")},
+        ]
+        # If the question is superlative-oriented, encourage ordering
+        ql = question.lower()
+        if any(w in ql for w in ["most", "largest", "highest", "fewest", "least"]):
+            inv.append({"name": "order_by_count", "description": "ORDER BY employee_count DESC (or ASC for fewest/least)", "check": lambda s: ("fewest" in ql or "least" in ql and re.search(r"order\s+by\s+employee_count\s+asc", s, re.IGNORECASE)) or re.search(r"order\s+by\s+employee_count\s+desc", s, re.IGNORECASE)})
+    elif category == CATEGORY_SIMPLE_COUNT_EMP:
+        inv += [
+            {"name": "from_employee", "description": "FROM employee table", "check": _regex(r"from\s+employee")},
+            {"name": "count_all", "description": "COUNT(*) with alias total_employees", "check": _regex(r"count\s*\(\s*\*\s*\)\s+as\s+total_employees")},
+            {"name": "limit_100", "description": "Ends with LIMIT 100", "check": _regex(r"limit\s+100;?\s*$")},
+        ]
+    elif category == CATEGORY_SIMPLE_COUNT_EMP_CURRENT:
+        inv += [
+            {"name": "join_dept_emp", "description": "Join dept_emp de", "check": _regex(r"join\s+dept_emp")},
+            {"name": "current_filter", "description": "de.to_date='9999-01-01' filter", "check": _regex(r"de\.to_date\s*=\s*'9999-01-01'")},
+            {"name": "count_distinct_emp", "description": "COUNT(DISTINCT de.emp_no) alias current_employee_count", "check": _regex(r"count\s*\(\s*distinct\s+de\.emp_no\s*\)\s+as\s+current_employee_count")},
+            {"name": "limit_100", "description": "Ends with LIMIT 100", "check": _regex(r"limit\s+100;?\s*$")},
+        ]
+    elif category == CATEGORY_SIMPLE_COUNT_DEPT:
+        inv += [
+            {"name": "from_department", "description": "FROM department", "check": _regex(r"from\s+department")},
+            {"name": "count_depts", "description": "COUNT(*) AS department_count", "check": _regex(r"count\s*\(\s*\*\s*\)\s+as\s+department_count")},
+            {"name": "limit_100", "description": "Ends with LIMIT 100", "check": _regex(r"limit\s+100;?\s*$")},
+        ]
+    elif category == CATEGORY_SALARY_EXTREMES_GLOBAL:
+        # Extremely strict invariants for highest/lowest salary by department
+        inv += [
+            {"name": "from_salary_s", "description": "FROM salary s", "check": _regex(r"from\s+salary\s+s")},
+            {"name": "join_dept_emp_de", "description": "JOIN dept_emp de ON s.emp_no = de.emp_no", "check": _regex(r"join\s+dept_emp\s+de\s+on\s+s\.emp_no\s*=\s*de\.emp_no")},
+            {"name": "join_department_d", "description": "JOIN department d ON de.dept_no = d.dept_no", "check": _regex(r"join\s+department\s+d\s+on\s+de\.dept_no\s*=\s*d\.dept_no")},
+            {"name": "select_dept_name", "description": "SELECT d.dept_name", "check": _regex(r"select\s+d\.dept_name")},
+            {"name": "select_max_min_salary", "description": "SELECT MAX(s.salary) AS highest_salary, MIN(s.salary) AS lowest_salary", "check": lambda s: re.search(r"max\s*\(\s*s\.salary\s*\)\s+as\s+highest_salary", s, re.IGNORECASE) and re.search(r"min\s*\(\s*s\.salary\s*\)\s+as\s+lowest_salary", s, re.IGNORECASE)},
+            {"name": "group_by_dept_name", "description": "GROUP BY d.dept_name", "check": _regex(r"group\s+by\s+d\.dept_name")},
+            {"name": "limit_100", "description": "Ends with LIMIT 100", "check": _regex(r"limit\s+100;?\s*$")},
+            {"name": "no_unaliased_salary", "description": "Do not use MAX(salary) or MIN(salary) without s. alias", "check": lambda s: not re.search(r"max\s*\(\s*salary\s*\)", s, re.IGNORECASE) and not re.search(r"min\s*\(\s*salary\s*\)", s, re.IGNORECASE)},
+            {"name": "no_unaliased_dept_name", "description": "Do not use dept_name without d. alias", "check": lambda s: not re.search(r"[^a-zA-Z0-9_]dept_name", s) or re.search(r"d\.dept_name", s)},
+        ]
+    elif category == CATEGORY_SALARY_RANGE_BY_DEPT:
+        # Stricter invariants for salary range by department
+        inv += [
+            {"name": "joins_salary", "description": "FROM salary s", "check": _regex(r"from\s+salary\s+s")},
+            {"name": "joins_dept_emp", "description": "Join dept_emp de", "check": _regex(r"join\s+dept_emp")},
+            {"name": "joins_department", "description": "Join department d", "check": _regex(r"join\s+department")},
+            {"name": "select_dept_name", "description": "Select d.dept_name", "check": _regex(r"d\.dept_name")},
+            {"name": "min_max", "description": "MIN(s.salary) & MAX(s.salary)", "check": lambda s: re.search(r"min\s*\(\s*s\.salary", s, re.IGNORECASE) and re.search(r"max\s*\(\s*s\.salary", s, re.IGNORECASE)},
+            {"name": "dept_filter", "description": "Filter on d.dept_name=...", "check": _regex(r"d\.dept_name\s*=\s*'")},
+            {"name": "group_by_dept", "description": "GROUP BY d.dept_name", "check": _regex(r"group\s+by\s+.*d\.dept_name")},
+            {"name": "limit_100", "description": "Ends with LIMIT 100", "check": _regex(r"limit\s+100;?\s*$")},
+        ]
+    elif category == CATEGORY_TOP_N_SALARIES:
+        inv += [
+            {"name": "joins_salary", "description": "FROM salary s", "check": _regex(r"from\s+salary\s+s")},
+            {"name": "current_filter", "description": "s.to_date='9999-01-01'", "check": _regex(r"s\.to_date\s*=\s*'9999-01-01'")},
+            {"name": "order_desc", "description": "ORDER BY s.salary DESC", "check": _regex(r"order\s+by\s+s\.salary\s+desc")},
+            {"name": "limit_present", "description": "Has a LIMIT clause", "check": _regex(r"limit\s+\d+")},
+        ]
+    elif category == CATEGORY_DEPT_MANAGER_LIST:
+        inv += [
+            {"name": "joins_dept_manager", "description": "Join dept_manager dm", "check": _regex(r"join\s+dept_manager")},
+            {"name": "joins_employee", "description": "Join employee e", "check": _regex(r"join\s+employee")},
+            {"name": "select_names", "description": "Select manager first_name / last_name", "check": lambda s: "first_name" in s.lower() and "last_name" in s.lower()},
+            {"name": "dept_name", "description": "Select dept_name", "check": _regex(r"dept_name")},
+            {"name": "current_filter", "description": "dm.to_date='9999-01-01'", "check": _regex(r"dm\.to_date\s*=\s*'9999-01-01'")},
+            {"name": "limit_100", "description": "Ends with LIMIT 100", "check": _regex(r"limit\s+100;?\s*$")},
+        ]
+    elif category == CATEGORY_MANAGER_TENURE:
+        inv += [
+            {"name": "joins_dept_manager", "description": "Join dept_manager dm", "check": _regex(r"join\s+dept_manager")},
+            {"name": "current_filter", "description": "dm.to_date='9999-01-01'", "check": _regex(r"dm\.to_date\s*=\s*'9999-01-01'")},
+            {"name": "tenure_expr", "description": "Contains dm.from_date for tenure calc", "check": _regex(r"dm\.from_date")},
+            {"name": "limit_100", "description": "Ends with LIMIT 100", "check": _regex(r"limit\s+100;?\s*$")},
+        ]
+    elif category == CATEGORY_MANAGER_CHANGE_FREQUENCY:
+        inv += [
+            {"name": "from_dept_manager", "description": "FROM dept_manager", "check": _regex(r"from\s+dept_manager")},
+            {"name": "group_by_dept", "description": "GROUP BY dept_no or dept", "check": _regex(r"group\s+by\s+.*dept")},
+            {"name": "count_changes", "description": "COUNT(*) or COUNT(DISTINCT emp_no)", "check": _regex(r"count\s*\(")},
+            {"name": "order_desc", "description": "ORDER BY count desc", "check": _regex(r"order\s+by\s+.*count.*desc")},
+            {"name": "limit_100", "description": "Ends with LIMIT 100", "check": _regex(r"limit\s+100;?\s*$")},
+        ]
+    elif category == CATEGORY_TITLE_DISTINCT_COUNT:
+        inv += [
+            {"name": "from_title", "description": "FROM title t", "check": _regex(r"from\s+title")},
+            {"name": "count_distinct_titles", "description": "COUNT(DISTINCT t.title) alias title_count", "check": _regex(r"count\s*\(\s*distinct\s+t\.title\s*\)\s+as\s+title_count")},
+            {"name": "limit_100", "description": "Ends with LIMIT 100", "check": _regex(r"limit\s+100;?\s*$")},
+        ]
+    elif category == CATEGORY_TITLE_SPECIFIC_COUNT:
+        inv += [
+            {"name": "from_title", "description": "FROM title t", "check": _regex(r"from\s+title")},
+            {"name": "title_filter", "description": "Filter t.title='Senior Engineer'", "check": _regex(r"t\.title\s*=\s*'senior engineer'")},
+            {"name": "current_filter", "description": "t.to_date='9999-01-01'", "check": _regex(r"t\.to_date\s*=\s*'9999-01-01'")},
+            {"name": "count_distinct_emp", "description": "COUNT(DISTINCT t.emp_no)", "check": _regex(r"count\s*\(\s*distinct\s+t\.emp_no")},
+            {"name": "limit_100", "description": "Ends with LIMIT 100", "check": _regex(r"limit\s+100;?\s*$")},
+        ]
+    elif category == CATEGORY_TITLE_DISTRIBUTION_BY_DEPT:
+        inv += [
+            {"name": "joins_title", "description": "Join title t", "check": _regex(r"join\s+title")},
+            {"name": "joins_dept_emp", "description": "Join dept_emp de", "check": _regex(r"join\s+dept_emp")},
+            {"name": "joins_department", "description": "Join department d", "check": _regex(r"join\s+department")},
+            {"name": "current_filters", "description": "t.to_date & de.to_date current filters", "check": lambda s: "t.to_date='9999-01-01'" in s.lower() and "de.to_date='9999-01-01'" in s.lower()},
+            {"name": "group_by", "description": "GROUP BY dept and title", "check": _regex(r"group\s+by\s+.*(dept|title).*(title|dept)")},
+            {"name": "limit_100", "description": "Ends with LIMIT 100", "check": _regex(r"limit\s+100;?\s*$")},
+        ]
+    elif category == CATEGORY_PROMOTION_PATH_ENGINEER_TO_SENIOR:
+        inv += [
+            {"name": "dual_title_refs", "description": "References title table twice (t1, t2)", "check": _regex(r"title\s+t1.*title\s+t2|title\s+t2.*title\s+t1")},
+            {"name": "promotion_condition", "description": "t1.title='Engineer' AND t2.title='Senior Engineer'", "check": _regex(r"t1\.title\s*=\s*'engineer'.*t2\.title\s*=\s*'senior engineer'|t2\.title\s*=\s*'senior engineer'.*t1\.title\s*=\s*'engineer'")},
+            {"name": "temporal_order", "description": "t1.from_date < t2.from_date", "check": _regex(r"t1\.from_date\s*<\s*t2\.from_date")},
+            {"name": "distinct_emp", "description": "DISTINCT emp_no selected", "check": _regex(r"select\s+distinct\s+t1\.emp_no|select\s+distinct\s+t2\.emp_no")},
+            {"name": "limit_100", "description": "Ends with LIMIT 100", "check": _regex(r"limit\s+100;?\s*$")},
+        ]
+    elif category == CATEGORY_EMP_TENURE_CURRENT_DEPT:
+        inv += [
+            {"name": "joins_dept_emp", "description": "FROM dept_emp de", "check": _regex(r"from\s+dept_emp")},
+            {"name": "current_filter", "description": "de.to_date='9999-01-01'", "check": _regex(r"de\.to_date\s*=\s*'9999-01-01'")},
+            {"name": "tenure_expr", "description": "Uses de.from_date", "check": _regex(r"de\.from_date")},
+            {"name": "limit_100", "description": "Ends with LIMIT 100", "check": _regex(r"limit\s+100;?\s*$")},
+        ]
+    elif category == CATEGORY_HIRING_TREND_BY_YEAR:
+        inv += [
+            {"name": "from_employee", "description": "FROM employee e", "check": _regex(r"from\s+employee")},
+            {"name": "year_extract", "description": "EXTRACT(YEAR FROM hire_date)", "check": _regex(r"extract\s*\(\s*year\s+from\s+.*hire_date")},
+            {"name": "group_by_year", "description": "GROUP BY year", "check": _regex(r"group\s+by\s+.*year")},
+            {"name": "limit_100", "description": "Ends with LIMIT 100", "check": _regex(r"limit\s+100;?\s*$")},
+        ]
+    elif category == CATEGORY_SALARY_TREND_BY_YEAR:
+        inv += [
+            {"name": "from_salary", "description": "FROM salary s", "check": _regex(r"from\s+salary")},
+            {"name": "year_extract", "description": "EXTRACT(YEAR FROM s.from_date)", "check": _regex(r"extract\s*\(\s*year\s+from\s+s\.from_date")},
+            {"name": "avg_salary", "description": "AVG(s.salary)", "check": _regex(r"avg\s*\(\s*s\.salary")},
+            {"name": "group_by_year", "description": "GROUP BY year", "check": _regex(r"group\s+by\s+.*year")},
+            {"name": "limit_100", "description": "Ends with LIMIT 100", "check": _regex(r"limit\s+100;?\s*$")},
+        ]
+    elif category == CATEGORY_EMP_DEPT_CHANGES:
+        inv += [
+            {"name": "from_dept_emp", "description": "FROM dept_emp de", "check": _regex(r"from\s+dept_emp")},
+            {"name": "group_by_emp", "description": "GROUP BY de.emp_no", "check": _regex(r"group\s+by\s+.*emp_no")},
+            {"name": "having_multiple", "description": "HAVING COUNT(DISTINCT dept_no) > 1", "check": _regex(r"having\s+count\s*\(\s*distinct\s+dept_no\s*\)\s*>\s*1")},
+            {"name": "limit_100", "description": "Ends with LIMIT 100", "check": _regex(r"limit\s+100;?\s*$")},
+        ]
+    elif category == CATEGORY_GENDER_DIST_BY_DEPT:
+        inv += [
+            {"name": "joins_employee", "description": "Join employee e", "check": _regex(r"join\s+employee")},
+            {"name": "joins_dept_emp", "description": "Join dept_emp de", "check": _regex(r"join\s+dept_emp")},
+            {"name": "joins_department", "description": "Join department d", "check": _regex(r"join\s+department")},
+            {"name": "current_filter", "description": "de.to_date='9999-01-01'", "check": _regex(r"de\.to_date\s*=\s*'9999-01-01'")},
+            {"name": "group_by_dept_gender", "description": "GROUP BY dept & gender", "check": _regex(r"group\s+by\s+.*(dept|gender).*(gender|dept)")},
+            {"name": "limit_100", "description": "Ends with LIMIT 100", "check": _regex(r"limit\s+100;?\s*$")},
+        ]
+    elif category == CATEGORY_GENDER_PAY_GAP_BY_DEPT:
+        inv += [
+            {"name": "joins_salary", "description": "Join salary s", "check": _regex(r"join\s+salary")},
+            {"name": "joins_employee", "description": "Join employee e", "check": _regex(r"join\s+employee")},
+            {"name": "joins_dept_emp", "description": "Join dept_emp de", "check": _regex(r"join\s+dept_emp")},
+            {"name": "joins_department", "description": "Join department d", "check": _regex(r"join\s+department")},
+            {"name": "current_filters", "description": "Current rows s.to_date & de.to_date", "check": lambda s: "s.to_date='9999-01-01'" in s.lower() and "de.to_date='9999-01-01'" in s.lower()},
+            {"name": "avg_salary", "description": "AVG(s.salary)", "check": _regex(r"avg\s*\(\s*s\.salary")},
+            {"name": "group_by_dept_gender", "description": "GROUP BY dept & gender", "check": _regex(r"group\s+by\s+.*(dept|gender).*(gender|dept)")},
+            {"name": "limit_100", "description": "Ends with LIMIT 100", "check": _regex(r"limit\s+100;?\s*$")},
+        ]
+    elif category == CATEGORY_FEMALE_MANAGERS_COUNT:
+        inv += [
+            {"name": "joins_dept_manager", "description": "Join dept_manager dm", "check": _regex(r"join\s+dept_manager")},
+            {"name": "joins_employee", "description": "Join employee e", "check": _regex(r"join\s+employee")},
+            {"name": "female_filter", "description": "e.gender='F'", "check": _regex(r"e\.gender\s*=\s*'f'")},
+            {"name": "current_filter", "description": "dm.to_date='9999-01-01'", "check": _regex(r"dm\.to_date\s*=\s*'9999-01-01'")},
+            {"name": "count_distinct_mgr", "description": "COUNT(DISTINCT dm.emp_no)", "check": _regex(r"count\s*\(\s*distinct\s+dm\.emp_no")},
+            {"name": "limit_100", "description": "Ends with LIMIT 100", "check": _regex(r"limit\s+100;?\s*$")},
+        ]
+    elif category == CATEGORY_AVG_SALARY_BY_TITLE_AND_DEPT:
+        inv += [
+            {"name": "joins_salary", "description": "Join salary s", "check": _regex(r"join\s+salary")},
+            {"name": "joins_title", "description": "Join title t", "check": _regex(r"join\s+title")},
+            {"name": "joins_dept_emp", "description": "Join dept_emp de", "check": _regex(r"join\s+dept_emp")},
+            {"name": "joins_department", "description": "Join department d", "check": _regex(r"join\s+department")},
+            {"name": "current_filters", "description": "Current rows for s,de,t", "check": lambda s: all(x in s.lower() for x in ["s.to_date='9999-01-01'", "de.to_date='9999-01-01'", "t.to_date='9999-01-01'"])},
+            {"name": "avg_salary", "description": "AVG(s.salary)", "check": _regex(r"avg\s*\(\s*s\.salary")},
+            {"name": "group_by", "description": "GROUP BY dept & title", "check": _regex(r"group\s+by\s+.*(dept|title).*(title|dept)")},
+            {"name": "limit_100", "description": "Ends with LIMIT 100", "check": _regex(r"limit\s+100;?\s*$")},
+        ]
+    elif category == CATEGORY_GENDER_MANAGEMENT_COUNT:
+        inv += [
+            {"name": "joins_dept_manager", "description": "Join dept_manager dm", "check": _regex(r"join\s+dept_manager")},
+            {"name": "joins_employee", "description": "Join employee e", "check": _regex(r"join\s+employee")},
+            {"name": "current_filter", "description": "dm.to_date='9999-01-01'", "check": _regex(r"dm\.to_date\s*=\s*'9999-01-01'")},
+            {"name": "group_by_gender", "description": "GROUP BY gender", "check": _regex(r"group\s+by\s+.*gender")},
+            {"name": "count_mgr", "description": "COUNT(*) or COUNT(dm.emp_no)", "check": _regex(r"count\s*\(")},
+            {"name": "limit_100", "description": "Ends with LIMIT 100", "check": _regex(r"limit\s+100;?\s*$")},
+        ]
+    elif category == CATEGORY_SENIOR_STAFF_SUPERLATIVE:
+        inv += [
+            {"name": "joins_title", "description": "Join title t", "check": _regex(r"join\s+title")},
+            {"name": "joins_dept_emp", "description": "Join dept_emp de", "check": _regex(r"join\s+dept_emp")},
+            {"name": "joins_department", "description": "Join department d", "check": _regex(r"join\s+department")},
+            {"name": "senior_filter", "description": "t.title LIKE 'Senior%'", "check": _regex(r"t\.title\s+like\s+'senior%'")},
+            {"name": "current_filters", "description": "Current rows de.to_date & t.to_date", "check": lambda s: "de.to_date='9999-01-01'" in s.lower() and "t.to_date='9999-01-01'" in s.lower()},
+            {"name": "group_by_dept", "description": "GROUP BY d.dept_name", "check": _regex(r"group\s+by\s+.*dept_name")},
+            {"name": "order_desc", "description": "ORDER BY count desc", "check": _regex(r"order\s+by\s+.*count.*desc")},
+            {"name": "limit_100", "description": "Ends with LIMIT 100", "check": _regex(r"limit\s+100;?\s*$")},
+        ]
+    return inv
+
+# --------------- Enforcement -----------------
+
+def evaluate_invariants(sql: str, invariants: List[Dict[str, any]]) -> List[Dict[str, any]]:
+    failures = []
+    for inv in invariants:
+        try:
+            ok = inv['check'](sql)
+        except Exception:
+            ok = False
+        if not ok:
+            failures.append(inv)
+    return failures
+
+
+# More explicit prompt for join-based salary extremes queries
+REGEN_PROMPT_TEMPLATE = (
+    "You produced SQL for the user question. Some required invariants are missing.\n"
+    "Regenerate a SINGLE valid PostgreSQL SQL query that satisfies ALL listed invariants.\n"
+    "Return ONLY the SQL (no commentary). Must end with LIMIT 100; and a semicolon.\n"
+    "If the question is about highest/lowest salary by department, you MUST:\n"
+    "- Use FROM salary s\n"
+    "- JOIN dept_emp de ON s.emp_no = de.emp_no\n"
+    "- JOIN department d ON de.dept_no = d.dept_no\n"
+    "- SELECT d.dept_name, MAX(s.salary) AS highest_salary, MIN(s.salary) AS lowest_salary\n"
+    "- GROUP BY d.dept_name\n"
+    "- Use correct table aliases everywhere (no unaliased salary or dept_name)\n"
+    "- Do NOT reference columns that do not exist in the schema\n"
+    "User Question: {question}\n"
+    "Previous Attempt: {previous_sql}\n"
+    "Missing Invariants:\n{invariant_lines}\n"
+    "Regenerate now:"
+)
+
+
+def regenerate_with_constraints(question: str, previous_sql: str, missing: List[Dict[str, any]], llm, run_llm_fn, attempt: int) -> Optional[str]:
+    invariant_lines = "\n".join(f"- {m['name']}: {m['description']}" for m in missing)
+    prompt = REGEN_PROMPT_TEMPLATE.format(question=question, previous_sql=previous_sql, invariant_lines=invariant_lines)
+    # Use proper timeout values instead of hardcoded ones
+    # Start with a good timeout and decrease gradually for subsequent attempts
+    base_timeout = float(os.environ.get("INVARIANTS_BASE_TIMEOUT", 90))  # Start with 90s
+    timeout = max(30.0, base_timeout - attempt * 20.0)  # Decrease by 20s per attempt, minimum 30s
+    try:
+        raw = run_llm_fn(llm, prompt, timeout=max(8.0, timeout), retries=0)
+        # Normalize raw response to text
+        raw_text = getattr(raw, 'text', None)
+        if raw_text is None:
+            raw_text = str(raw)
+        # Extract first SELECT ... ;
+        m = re.search(r"(SELECT[\s\S]+?;)", raw_text, flags=re.IGNORECASE)
+        candidate = m.group(1) if m else raw_text.strip()
+        # Clean markdown fences
+        candidate = re.sub(r"```(?:sql)?|```", "", candidate, flags=re.IGNORECASE).strip()
+        # Ensure single statement & required LIMIT 100 at end
+        # Collapse accidental mid-statement semicolon before LIMIT
+        candidate = re.sub(r";\s*(LIMIT\s+\d+)", r" \1", candidate, flags=re.IGNORECASE)
+        # Append LIMIT 100 if missing (invariants will still verify)
+        if 'limit' not in candidate.lower():
+            if candidate.endswith(';'):
+                candidate = candidate[:-1] + ' LIMIT 100;'
+            else:
+                candidate += ' LIMIT 100;'
+        # Guarantee trailing semicolon
+        if not candidate.strip().endswith(';'):
+            candidate += ';'
+        return candidate
+    except Exception as e:
+        print(f"[INVARIANTS] Regeneration failed: {e}")
+        return None
+
+
+def enforce_invariants(question: str, sql: str, llm, run_llm_fn, max_regen: int = 2) -> Dict[str, any]:
+    category = classify_question(question)
+    if category == CATEGORY_GENERIC:
+        return {"category": category, "final_sql": sql, "regenerations": []}
+    invariants = build_invariants(category, question)
+    regenerations = []
+    current_sql = sql
+    # Increase max_regen to 5 for more LLM attempts
+    for attempt in range(max_regen + 4):
+        missing = evaluate_invariants(current_sql, invariants)
+        if not missing:
+            return {"category": category, "final_sql": current_sql, "regenerations": regenerations}
+        if attempt == max_regen:
+            # Give up, return last with missing list for transparency
+            return {"category": category, "final_sql": current_sql, "regenerations": regenerations, "remaining_missing": [m['name'] for m in missing]}
+        regen_sql = regenerate_with_constraints(question, current_sql, missing, llm, run_llm_fn, attempt)
+        if not regen_sql:
+            return {"category": category, "final_sql": current_sql, "regenerations": regenerations, "remaining_missing": [m['name'] for m in missing]}
+        regenerations.append({
+            "attempt": attempt + 1,
+            "missing_before": [m['name'] for m in missing],
+            "sql": regen_sql
+        })
+        current_sql = regen_sql
+    return {"category": category, "final_sql": current_sql, "regenerations": regenerations}

Agent/llm_health_check.py

@@ -0,0 +1,36 @@
+import os, time, json, sys
+from llama_index.llms.ollama import Ollama
+
+BASE_URL = os.environ.get("OLLAMA_BASE_URL", "http://localhost:11434")
+MODEL = os.environ.get("FINETUNED_MODEL_NAME") or os.environ.get("OLLAMA_MODEL_NAME", "gemma3:1b")
+
+print(f"Checking Ollama LLM connectivity: base_url={BASE_URL} model={MODEL}")
+
+# Lightweight client check without llama_index first (raw HTTP) to see if daemon responds
+try:
+    import requests
+    r = requests.get(f"{BASE_URL.rstrip('/')}/api/tags", timeout=5)
+    print(f"/api/tags status: {r.status_code}")
+    if r.ok:
+        tags = r.json().get('models', [])
+        names = [m.get('name') for m in tags]
+        print(f"Available models: {names}")
+        if MODEL not in names:
+            print(f"Model '{MODEL}' not yet pulled. Run: ollama pull {MODEL}")
+    else:
+        print("Non-200 response from Ollama daemon")
+except Exception as e:
+    print(f"Raw HTTP check failed: {e}")
+
+# LlamaIndex call
+try:
+    llm = Ollama(model=MODEL, base_url=BASE_URL, temperature=0.1, request_timeout=30)
+    t0 = time.time()
+    resp = llm.complete(prompt="SELECT 1;")
+    dt = time.time() - t0
+    print(f"LLM call success in {dt:.2f}s. Raw text: {resp.text[:120]!r}")
+except Exception as e:
+    print(f"LLM complete() failed: {type(e).__name__}: {e}")
+    sys.exit(1)
+
+print("Health check finished.")

Agent/mcp_client.py

@@ -0,0 +1,48 @@
+"""Minimal MCP client for local MCP server.
+
+Provides mcp_complete(prompt, timeout, context) -> str
+Tries to POST to MCP server and return the completion text. Raises on errors so caller can fallback.
+"""
+import os
+import json
+import requests
+from typing import Any, Dict, Optional
+
+MCP_URL = os.environ.get("MCP_URL", "http://127.0.0.1:8701/complete")
+
+
+def mcp_complete(prompt: str, timeout: float = 30.0, context: Optional[Dict[str, Any]] = None) -> str:
+    """Send a completion request to the local MCP server and return text.
+
+    Raises requests.RequestException on transport errors or ValueError if MCP returns error.
+    """
+    payload = {
+        "prompt": prompt,
+        "context": context or {},
+        "timeout": timeout
+    }
+    headers = {"Content-Type": "application/json"}
+    resp = requests.post(MCP_URL, data=json.dumps(payload), headers=headers, timeout=timeout)
+    resp.raise_for_status()
+    body = resp.json()
+    # Expecting {'text': '...'} from MCP server
+    if not isinstance(body, dict) or 'text' not in body:
+        raise ValueError(f"Unexpected MCP response: {body}")
+    return body['text']
+
+
+def mcp_call_tool(prompt: str, timeout: float = 60.0, context: Optional[Dict[str, Any]] = None) -> Dict[str, Any]:
+    """Call the MCP /complete endpoint and return the full JSON response.
+
+    This is useful when asking MCP to run a registered tool; the tool result will
+    appear under the `tool_result` key in the returned dict.
+    """
+    payload = {
+        "prompt": prompt,
+        "context": context or {},
+        "timeout": timeout,
+    }
+    headers = {"Content-Type": "application/json"}
+    resp = requests.post(MCP_URL, data=json.dumps(payload), headers=headers, timeout=timeout)
+    resp.raise_for_status()
+    return resp.json()

Agent/mcp_server.py

@@ -0,0 +1,503 @@
+from fastapi import FastAPI, HTTPException
+from pydantic import BaseModel
+from typing import Any, Dict, Optional, List
+import asyncio
+import importlib
+import uuid
+import os
+import re
+
+# Attempt to import agent workflow for schema/inspector & utilities (optional)
+try:
+    import Agent.agentic_workflow as aw
+    print(f"[MCP STARTUP] Successfully imported Agent.agentic_workflow")
+except Exception as e:
+    print(f"[MCP STARTUP] Failed to import Agent.agentic_workflow: {e}")
+    aw = None
+
+app = FastAPI(title="Minimal MCP Server (dev)")
+
+# Simple tool registry
+TOOLS: Dict[str, Any] = {}
+
+
+class CompleteRequest(BaseModel):
+    prompt: str
+    context: Optional[Dict[str, Any]] = None
+    timeout: Optional[float] = 30.0
+
+
+def register_tool(name: str, fn: Any):
+    TOOLS[name] = fn
+
+
+async def _call_tool(name: str, args: Dict[str, Any]) -> Any:
+    if name not in TOOLS:
+        raise KeyError(f"Tool '{name}' not registered")
+    fn = TOOLS[name]
+    if asyncio.iscoroutinefunction(fn):
+        return await fn(**(args or {}))
+    else:
+        # run sync in threadpool
+        loop = asyncio.get_running_loop()
+        return await loop.run_in_executor(None, lambda: fn(**(args or {})))
+
+
+@app.on_event("startup")
+async def _startup():
+    # Register built-in/mock tools lazily to avoid importing heavy modules at import time
+    # Try to register a real document scanner implementation; otherwise register a mock
+    ds_registered = None
+    try:
+        from .tools import document_scanner as ds
+        register_tool("document_scanner.process", ds.process_image)
+        ds_registered = f"Agent.tools.document_scanner.process_image (relative import)"
+    except Exception:
+        # If relative import fails (running as script), try absolute import
+        try:
+            import Agent.tools.document_scanner as ds2
+            register_tool("document_scanner.process", ds2.process_image)
+            ds_registered = f"Agent.tools.document_scanner.process_image (absolute import)"
+        except Exception:
+            # Register a fallback mock tool
+            def _mock_doc(**kwargs):
+                # Accept arbitrary keyword args (image_url, image_bytes, etc.) to be tolerant
+                return {"doc_id": str(uuid.uuid4()), "text": "(mock OCR output)", "metadata": {"pages": 1, "confidence": 0.0}, "received_args": kwargs}
+            register_tool("document_scanner.process", _mock_doc)
+            ds_registered = "mock(document_scanner.process)"
+
+    # Log which document scanner implementation was registered for easier debugging
+    try:
+        print(f"[MCP STARTUP] Registered document_scanner.process -> {ds_registered}")
+    except Exception:
+        pass
+
+    # Register Granite Vision direct visual Q&A tool
+    gv_registered = None
+    try:
+        from .tools import granite_vision as gv
+        register_tool("granite_vision.qa", gv.qa)
+        gv_registered = "Agent.tools.granite_vision.qa (relative import)"
+    except Exception as e1:
+        print(f"[MCP STARTUP] granite_vision relative import FAILED: {type(e1).__name__}: {e1}")
+        try:
+            import Agent.tools.granite_vision as gv2
+            register_tool("granite_vision.qa", gv2.qa)
+            gv_registered = "Agent.tools.granite_vision.qa (absolute import)"
+        except Exception as e2:
+            print(f"[MCP STARTUP] granite_vision absolute import FAILED: {type(e2).__name__}: {e2}")
+            def _mock_gv(**kwargs):
+                return {"answer": "(granite_vision unavailable - mock)", "model": "mock", "received_args": kwargs}
+            register_tool("granite_vision.qa", _mock_gv)
+            gv_registered = "mock(granite_vision.qa)"
+    print(f"[MCP STARTUP] Registered granite_vision.qa -> {gv_registered}")
+
+
+    # Register OCR QA composite tool: runs OCR then runs the agent OCR QA helper
+    try:
+        # Use the global aw variable instead of re-importing
+        if not aw:
+            raise ImportError("Agent workflow not available")
+
+        async def _ocr_qa_tool(image_url: Optional[str] = None, image_bytes: Optional[bytes] = None, question: Optional[str] = None, **kwargs):
+            # Call the document scanner tool first (use the registered one to allow mocks)
+            ds_fn = TOOLS.get("document_scanner.process")
+            if not ds_fn:
+                raise RuntimeError("document_scanner.process not registered")
+
+            loop = asyncio.get_running_loop()
+            try:
+                if asyncio.iscoroutinefunction(ds_fn):
+                    tool_res = await ds_fn(image_url=image_url, image_bytes=image_bytes, options=kwargs)
+                else:
+                    tool_res = await loop.run_in_executor(None, lambda: ds_fn(image_url=image_url, image_bytes=image_bytes, options=kwargs))
+            except Exception as e:
+                return {"error": f"OCR tool failed: {e}"}
+
+            doc_id = tool_res.get("doc_id") if isinstance(tool_res, dict) else None
+            ocr_text = (tool_res.get("text") if isinstance(tool_res, dict) else str(tool_res)) or ""
+
+            # Persist OCR text so downstream callers can retrieve it via doc_id
+            try:
+                if doc_id and ocr_text:
+                    docs_dir = os.path.join("chat_store", "docs")
+                    os.makedirs(docs_dir, exist_ok=True)
+                    doc_path = os.path.join(docs_dir, f"{doc_id}.txt")
+                    # Write file (overwrite if exists)
+                    with open(doc_path, "w", encoding="utf-8") as df:
+                        df.write(ocr_text)
+                    # Also add a short history entry if agent chat store available
+                    try:
+                        if hasattr(aw, 'chat_store_private') and hasattr(aw, 'ChatMessage') and hasattr(aw, 'MessageRole'):
+                            user_msg = aw.ChatMessage(role=aw.MessageRole.USER, content=f"Uploaded document {doc_id}")
+                            assistant_msg = aw.ChatMessage(role=aw.MessageRole.ASSISTANT, content=f"OCR stored: {ocr_text[:400]}")
+                            aw.chat_store_private.add_message(key="conversation", message=user_msg)
+                            aw.chat_store_private.add_message(key="conversation", message=assistant_msg)
+                            try:
+                                aw.chat_store_private.persist(str(aw.private_store_path))
+                            except Exception:
+                                pass
+                    except Exception:
+                        pass
+            except Exception:
+                # Non-fatal: continue even if persistence fails
+                pass
+
+            # If a question was provided, answer it using the agent helper (run in executor to avoid blocking)
+            if question:
+                try:
+                    # Use executor if helper is sync to avoid blocking event loop
+                    if callable(getattr(aw, 'ocr_agent_qa', None)):
+                        answer = await loop.run_in_executor(None, lambda: aw.ocr_agent_qa(question, doc_id))
+                        return {"doc_id": doc_id, "answer": answer}
+                    else:
+                        return {"doc_id": doc_id, "answer": "ocr_agent_qa not available"}
+                except Exception as e:
+                    return {"error": f"ocr_agent_qa failed: {e}", "tool_result": tool_res}
+
+            # No question: return the OCR result and doc_id
+            return {"doc_id": doc_id, "ocr_text": ocr_text}
+
+        register_tool("ocr_qa", _ocr_qa_tool)
+        print(f"[MCP STARTUP] Registered ocr_qa -> real _ocr_qa_tool (agent available)")
+    except Exception:
+        # If agent import fails, register a tolerant mock tool
+        def _ocr_qa_mock(**kwargs):
+            return {"doc_id": str(uuid.uuid4()), "answer": "(ocr_qa unavailable - mock response)", "received_args": kwargs}
+        register_tool("ocr_qa", _ocr_qa_mock)
+        print(f"[MCP STARTUP] Registered ocr_qa -> mock(ocr_qa)")
+
+    # ------------------------------------------------------------------
+    # Deterministic schema-aware SQL helper tools (agentic compliant)
+    # ------------------------------------------------------------------
+    ENABLE_FAST_SQL = os.environ.get("ENABLE_MCP_FAST_TOOLS", "1") in ("1","true","True")
+    if not ENABLE_FAST_SQL:
+        print("[MCP STARTUP] Fast SQL tools disabled via ENABLE_MCP_FAST_TOOLS=0")
+        return
+
+    # Initialize database connection if not already done
+    if not aw:
+        print("[MCP STARTUP][FAST] Agent workflow not available; skipping fast SQL tool registration.")
+        return
+        
+    if not getattr(aw, 'inspector', None) or not getattr(aw, 'available_tables', None):
+        print("[MCP STARTUP][FAST] Database not initialized; attempting to initialize...")
+        try:
+            # Initialize database connection - use await since we're in async context
+            db_uri = os.environ.get("DB_CONNECTION_URL")
+            if not db_uri:
+                print("[MCP STARTUP][FAST] DB_CONNECTION_URL not set; skipping fast SQL tool registration.")
+                return
+            db_result = await aw.initialize_database(db_uri)
+            if db_result.get("tables"):
+                print(f"[MCP STARTUP][FAST] Database initialized with {len(db_result['tables'])} tables")
+            else:
+                print("[MCP STARTUP][FAST] Database initialization failed; skipping fast SQL tool registration.")
+                return
+        except Exception as db_err:
+            print(f"[MCP STARTUP][FAST] Database initialization error: {db_err}; skipping fast SQL tool registration.")
+            return
+
+    inspector = aw.inspector
+    available_tables = aw.available_tables
+
+    def _colnames(table: str) -> List[str]:
+        try:
+            return [c['name'] for c in inspector.get_columns(table_name=table)]
+        except Exception:
+            return []
+
+    def _has_tables(*tables: str) -> bool:
+        at_lower = {t.lower() for t in available_tables}
+        return all(t.lower() in at_lower for t in tables)
+
+    # Simple validation & sanitization re-use
+    validate_sql_query = getattr(aw, 'validate_sql_query', lambda q: {"valid": True})
+    fix_common_sql_errors = getattr(aw, 'fix_common_sql_errors', lambda q: q)
+
+    async def _verify_sql(sql: str, question: str, tag: str) -> str:
+        """Optional light LLM verification (short prompt) if LLM available."""
+        if not aw or not getattr(aw, 'Settings', None) or not getattr(aw.Settings, 'llm', None):
+            return sql
+        try:
+            prompt = (
+                f"{aw.EMPLOYEE_DB_SCHEMA}\n\nYou are an SQL verifier. Tag={tag}."
+                " If the candidate SQL is valid and uses only existing tables/columns keep it."
+                " If SMALL fixes (alias, LIMIT 100, semicolon) needed, fix them. Never invent tables/columns."
+                f"\nQuestion: {question}\nCandidate SQL:\n{sql}\nReturn ONLY final SQL." )
+            resp = aw.Settings.llm.complete(prompt=prompt)
+            txt = getattr(resp, 'text', str(resp)).strip()
+            if '```' in txt:
+                parts = re.split(r"```(?:sql)?", txt, flags=re.IGNORECASE)
+                txt = '\n'.join(p for p in parts if 'select' in p.lower() or 'with' in p.lower())
+            if txt.lower().startswith('sql'):
+                txt = txt[3:].strip()
+            txt = fix_common_sql_errors(txt)
+            v = validate_sql_query(txt)
+            if v.get('valid'):
+                return txt
+        except Exception as ve:
+            print(f"[FAST][VERIFY][WARN] Verification failed: {ve}; using original SQL")
+        return sql
+
+    def _build_sql(select_cols: List[str], base_table: str, joins: Optional[List[str]] = None,
+                   where: Optional[List[str]] = None, group_by: Optional[List[str]] = None,
+                   order_by: Optional[str] = None, limit: Optional[int] = 100) -> str:
+        parts = ["SELECT " + ", ".join(select_cols), "FROM " + base_table]
+        for j in (joins or []):
+            parts.append(j)
+        if where:
+            parts.append("WHERE " + " AND ".join(where))
+        if group_by:
+            parts.append("GROUP BY " + ", ".join(group_by))
+        if order_by:
+            parts.append("ORDER BY " + order_by)
+        if limit is not None:
+            parts.append(f"LIMIT {limit}")
+        sql = " ".join(parts)
+        if not sql.strip().endswith(';'):
+            sql += ';'
+        return sql
+
+    async def tool_list_departments(question: str = ""):
+        if not _has_tables('department'):
+            return {"error": "department table not available"}
+        cols = _colnames('department')
+        name_col = 'dept_name' if 'dept_name' in cols else (cols[0] if cols else 'dept_name')
+        sql = _build_sql([f"d.{name_col} AS department_name"], 'department d', order_by='department_name ASC')
+        sql = fix_common_sql_errors(sql)
+        sql = await _verify_sql(sql, question or 'List departments', 'list_departments')
+        return {"sql": sql}
+
+    async def tool_count_employees(question: str = ""):
+        if not _has_tables('employee'):
+            return {"error": "employee table not available"}
+        sql = _build_sql(["COUNT(*) AS total_employees"], 'employee e', limit=None)
+        sql = fix_common_sql_errors(sql)
+        sql = await _verify_sql(sql, question or 'Total employees', 'count_employees')
+        return {"sql": sql}
+
+    async def tool_employee_count_by_department(question: str = ""):
+        if not _has_tables('department','dept_emp'):
+            return {"error": "required tables missing"}
+        dcols = _colnames('department'); decols = _colnames('dept_emp')
+        dept_name = 'dept_name' if 'dept_name' in dcols else (dcols[0] if dcols else 'dept_name')
+        dept_no = 'dept_no' if 'dept_no' in dcols or 'dept_no' in decols else 'dept_no'
+        emp_no = 'emp_no' if 'emp_no' in decols else 'emp_no'
+        where = []
+        if 'to_date' in decols:
+            where.append("de.to_date = '9999-01-01'")
+        sql = _build_sql([f"d.{dept_name} AS department_name", f"COUNT(DISTINCT de.{emp_no}) AS employee_count"],
+                         'department d', joins=[f"JOIN dept_emp de ON d.{dept_no} = de.{dept_no}"],
+                         where=where, group_by=[f"d.{dept_name}"], order_by='employee_count DESC')
+        sql = fix_common_sql_errors(sql)
+        sql = await _verify_sql(sql, question or 'Employees per department', 'employee_count_by_department')
+        return {"sql": sql}
+
+    async def tool_gender_pay_gap(question: str = ""):
+        if not _has_tables('department','dept_emp','employee','salary'):
+            return {"error": "required tables missing"}
+        scolumns = _colnames('salary')
+        sval = 'salary' if 'salary' in scolumns else ('amount' if 'amount' in scolumns else (scolumns[0] if scolumns else 'salary'))
+        where = ["de.to_date = '9999-01-01'", "s.to_date = '9999-01-01'"]
+        sql = _build_sql([
+            "d.dept_name",
+            f"AVG(CASE WHEN e.gender='M' THEN s.{sval} END) AS avg_male_salary",
+            f"AVG(CASE WHEN e.gender='F' THEN s.{sval} END) AS avg_female_salary",
+            f"(AVG(CASE WHEN e.gender='M' THEN s.{sval} END) - AVG(CASE WHEN e.gender='F' THEN s.{sval} END)) AS gap",
+            f"CASE WHEN AVG(CASE WHEN e.gender='M' THEN s.{sval} END)=0 THEN NULL ELSE ((AVG(CASE WHEN e.gender='M' THEN s.{sval} END) - AVG(CASE WHEN e.gender='F' THEN s.{sval} END))/NULLIF(AVG(CASE WHEN e.gender='M' THEN s.{sval} END),0))*100 END AS pct_gap"
+        ], 'department d',
+            joins=['JOIN dept_emp de ON d.dept_no = de.dept_no','JOIN employee e ON de.emp_no = e.emp_no','JOIN salary s ON e.emp_no = s.emp_no'],
+            where=where, group_by=['d.dept_name'], order_by='gap DESC')
+        sql = fix_common_sql_errors(sql)
+        sql = await _verify_sql(sql, question or 'Gender pay gap', 'gender_pay_gap')
+        return {"sql": sql}
+
+    async def tool_salary_extremes(question: str = ""):
+        if not _has_tables('salary'):
+            return {"error": "salary table missing"}
+        scolumns = _colnames('salary'); sval = 'salary' if 'salary' in scolumns else ('amount' if 'amount' in scolumns else (scolumns[0] if scolumns else 'salary'))
+        where = ["s.to_date = '9999-01-01'"] if 'to_date' in scolumns else []
+        sql = _build_sql([f"MAX(s.{sval}) AS highest_salary", f"MIN(s.{sval}) AS lowest_salary"], 'salary s', where=where, limit=None)
+        sql = fix_common_sql_errors(sql)
+        sql = await _verify_sql(sql, question or 'Salary extremes', 'salary_extremes')
+        return {"sql": sql}
+
+    async def tool_salary_range_department(department_name: str = "Development", question: str = ""):
+        if not _has_tables('department','dept_emp','salary'):
+            return {"error": "required tables missing"}
+        scolumns = _colnames('salary'); sval = 'salary' if 'salary' in scolumns else ('amount' if 'amount' in scolumns else (scolumns[0] if scolumns else 'salary'))
+        # Escape single quotes in department_name for SQL safety (basic)
+        escaped_dept = department_name.replace("'", "''")
+        where = ["de.to_date = '9999-01-01'", "s.to_date = '9999-01-01'", f"d.dept_name = '{escaped_dept}'"]
+        sql = _build_sql(["d.dept_name", f"MAX(s.{sval}) AS max_salary", f"MIN(s.{sval}) AS min_salary"], 'department d',
+                         joins=['JOIN dept_emp de ON d.dept_no = de.dept_no','JOIN salary s ON de.emp_no = s.emp_no'],
+                         where=where, group_by=['d.dept_name'], limit=None)
+        sql = fix_common_sql_errors(sql)
+        sql = await _verify_sql(sql, question or 'Department salary range', 'salary_range_department')
+        return {"sql": sql, "department": department_name}
+
+    async def tool_top_paid_employees(top_n: int = 10, question: str = ""):
+        if not _has_tables('employee','salary'):
+            return {"error": "required tables missing"}
+        scolumns = _colnames('salary'); sval = 'salary' if 'salary' in scolumns else (scolumns[0] if scolumns else 'salary')
+        where = ["s.to_date = '9999-01-01'"] if 'to_date' in scolumns else []
+        sql = _build_sql(["e.emp_no","e.first_name","e.last_name", f"s.{sval} AS salary"], 'employee e',
+                         joins=['JOIN salary s ON e.emp_no = s.emp_no'], where=where,
+                         order_by=f"s.{sval} DESC", limit=top_n)
+        sql = fix_common_sql_errors(sql)
+        sql = await _verify_sql(sql, question or 'Top paid employees', 'top_paid_employees')
+        return {"sql": sql, "top_n": top_n}
+
+    async def tool_department_highest_avg_salary(question: str = ""):
+        if not _has_tables('department','dept_emp','salary'):
+            return {"error": "required tables missing"}
+        scolumns = _colnames('salary'); sval = 'salary' if 'salary' in scolumns else (scolumns[0] if scolumns else 'salary')
+        where = ["de.to_date = '9999-01-01'", "s.to_date = '9999-01-01'"]
+        sql = _build_sql(["d.dept_name", f"AVG(s.{sval}) AS avg_salary"], 'department d',
+                         joins=['JOIN dept_emp de ON d.dept_no = de.dept_no','JOIN salary s ON de.emp_no = s.emp_no'],
+                         where=where, group_by=['d.dept_name'], order_by='avg_salary DESC', limit=1)
+        sql = fix_common_sql_errors(sql)
+        sql = await _verify_sql(sql, question or 'Department highest average salary', 'dept_highest_avg_salary')
+        return {"sql": sql}
+
+    async def tool_department_manager_listing(question: str = ""):
+        if not _has_tables('department','dept_manager','employee'):
+            return {"error": "required tables missing"}
+        where = ["dm.to_date = '9999-01-01'"]
+        sql = _build_sql(["d.dept_name AS department_name","e.first_name","e.last_name"], 'department d',
+                         joins=['JOIN dept_manager dm ON d.dept_no = dm.dept_no','JOIN employee e ON dm.emp_no = e.emp_no'],
+                         where=where, order_by='department_name ASC')
+        sql = fix_common_sql_errors(sql)
+        sql = await _verify_sql(sql, question or 'Department managers', 'department_manager_listing')
+        return {"sql": sql}
+
+    # NEW: Complex analytical tools for business intelligence
+    async def tool_gender_distribution_by_dept(question: str = ""):
+        if not _has_tables('department','dept_emp','employee'):
+            return {"error": "required tables missing"}
+        where = ["de.to_date = '9999-01-01'"]
+        sql = _build_sql([
+            "d.dept_name AS department_name",
+            "e.gender",
+            "COUNT(*) AS count",
+            "ROUND(COUNT(*) * 100.0 / SUM(COUNT(*)) OVER (PARTITION BY d.dept_name), 2) AS percentage"
+        ], 'department d',
+            joins=['JOIN dept_emp de ON d.dept_no = de.dept_no','JOIN employee e ON de.emp_no = e.emp_no'],
+            where=where, group_by=['d.dept_name', 'e.gender'], order_by='department_name, gender')
+        sql = fix_common_sql_errors(sql)
+        sql = await _verify_sql(sql, question or 'Gender distribution by department', 'gender_distribution_by_dept')
+        return {"sql": sql}
+
+    async def tool_hiring_trend_by_year(question: str = ""):
+        if not _has_tables('employee'):
+            return {"error": "required table employee missing"}
+        sql = _build_sql([
+            "EXTRACT(YEAR FROM e.hire_date) AS hire_year",
+            "COUNT(*) AS employees_hired",
+            "COUNT(CASE WHEN e.gender='M' THEN 1 END) AS male_hires",
+            "COUNT(CASE WHEN e.gender='F' THEN 1 END) AS female_hires"
+        ], 'employee e', group_by=['EXTRACT(YEAR FROM e.hire_date)'], order_by='hire_year ASC')
+        sql = fix_common_sql_errors(sql)
+        sql = await _verify_sql(sql, question or 'Hiring trends by year', 'hiring_trend_by_year')
+        return {"sql": sql}
+
+    async def tool_employee_tenure_by_dept(question: str = ""):
+        if not _has_tables('department','dept_emp','employee'):
+            return {"error": "required tables missing"}
+        where = ["de.to_date = '9999-01-01'"]
+        sql = _build_sql([
+            "d.dept_name AS department_name",
+            "AVG(EXTRACT(DAYS FROM (CURRENT_DATE - e.hire_date))/365.25) AS avg_tenure_years",
+            "MIN(EXTRACT(DAYS FROM (CURRENT_DATE - e.hire_date))/365.25) AS min_tenure_years", 
+            "MAX(EXTRACT(DAYS FROM (CURRENT_DATE - e.hire_date))/365.25) AS max_tenure_years"
+        ], 'department d',
+            joins=['JOIN dept_emp de ON d.dept_no = de.dept_no','JOIN employee e ON de.emp_no = e.emp_no'],
+            where=where, group_by=['d.dept_name'], order_by='avg_tenure_years DESC')
+        sql = fix_common_sql_errors(sql)
+        sql = await _verify_sql(sql, question or 'Employee tenure by department', 'employee_tenure_by_dept')
+        return {"sql": sql}
+
+    async def tool_female_managers_count(question: str = ""):
+        if not _has_tables('dept_manager','employee'):
+            return {"error": "required tables missing"}
+        where = ["dm.to_date = '9999-01-01'", "e.gender = 'F'"]
+        sql = _build_sql([
+            "COUNT(*) AS female_managers_count",
+            "COUNT(*) * 100.0 / (SELECT COUNT(*) FROM dept_manager dm2 WHERE dm2.to_date = '9999-01-01') AS percentage_female_managers"
+        ], 'dept_manager dm',
+            joins=['JOIN employee e ON dm.emp_no = e.emp_no'],
+            where=where, limit=None)
+        sql = fix_common_sql_errors(sql)
+        sql = await _verify_sql(sql, question or 'Female managers count', 'female_managers_count')
+        return {"sql": sql}
+
+    async def tool_avg_salary_by_title_dept(question: str = ""):
+        """Get average salary by title and department."""
+        if not _has_tables('department', 'dept_emp', 'salary', 'title', 'employee'):
+            return {"error": "required tables missing"}
+        where = ["de.to_date = '9999-01-01'", "s.to_date = '9999-01-01'", "t.to_date = '9999-01-01'"]
+        sql = _build_sql(["d.dept_name AS department_name", "t.title", "AVG(s.salary) AS avg_salary"], 'department d',
+                         joins=['JOIN dept_emp de ON d.dept_no = de.dept_no',
+                               'JOIN employee e ON de.emp_no = e.emp_no',
+                               'JOIN salary s ON e.emp_no = s.emp_no',
+                               'JOIN title t ON e.emp_no = t.emp_no'],
+                         where=where, group_by=['d.dept_name', 't.title'], 
+                         order_by='d.dept_name, avg_salary DESC', limit=100)
+        sql = fix_common_sql_errors(sql)
+        sql = await _verify_sql(sql, question or 'Average salary by title and department', 'avg_salary_by_title_dept')
+        return {"sql": sql}
+
+    register_tool('fast_sql.list_departments', tool_list_departments)
+    register_tool('fast_sql.count_employees', tool_count_employees)
+    register_tool('fast_sql.employee_count_by_department', tool_employee_count_by_department)
+    register_tool('fast_sql.gender_pay_gap', tool_gender_pay_gap)  # RE-ENABLED: Complex analytical queries
+    register_tool('fast_sql.salary_extremes', tool_salary_extremes)
+    register_tool('fast_sql.salary_range_department', tool_salary_range_department)
+    register_tool('fast_sql.top_paid_employees', tool_top_paid_employees)
+    register_tool('fast_sql.department_highest_avg_salary', tool_department_highest_avg_salary)
+    register_tool('fast_sql.department_manager_listing', tool_department_manager_listing)
+    
+    # Register new complex analytical tools
+    register_tool('fast_sql.gender_distribution_by_dept', tool_gender_distribution_by_dept)
+    register_tool('fast_sql.hiring_trend_by_year', tool_hiring_trend_by_year)
+    register_tool('fast_sql.employee_tenure_by_dept', tool_employee_tenure_by_dept)
+    register_tool('fast_sql.female_managers_count', tool_female_managers_count)
+    register_tool('fast_sql.avg_salary_by_title_dept', tool_avg_salary_by_title_dept)
+    print('[MCP STARTUP] Registered fast SQL tools: ' + ', '.join([k for k in TOOLS if k.startswith('fast_sql.')]))
+
+
+@app.post("/complete")
+async def complete(req: CompleteRequest):
+    """Minimal completion endpoint:
+    - If context contains {'tool_call': {'name': ..., 'args': {...}}}, the server will run the tool and return its result.
+    - Otherwise it returns a simple echo of the prompt under 'text'.
+    This scaffold is purposely small so you can extend tool orchestration later.
+    """
+    ctx = req.context or {}
+    # Tool invocation flow
+    if 'tool_call' in ctx:
+        tc = ctx['tool_call']
+        name = tc.get('name')
+        args = tc.get('args', {})
+        try:
+            tool_res = await _call_tool(name, args)
+            return {"text": f"Tool '{name}' executed.", "tool_result": tool_res}
+        except KeyError as ke:
+            raise HTTPException(status_code=404, detail=str(ke))
+        except Exception as e:
+            raise HTTPException(status_code=500, detail=f"Tool call failed: {e}")
+
+    # No tool requested: return a simple completion echo for now
+    # Future: integrate a local model here and tool loop orchestration
+    return {"text": req.prompt}
+
+
+@app.get("/tools")
+def list_tools():
+    return {"tools": list(TOOLS.keys())}
+
+
+@app.get("/health")
+def health():
+    return {"status": "ok", "tools_registered": len(TOOLS)}

Agent/memory_manager.py

@@ -0,0 +1,467 @@
+"""
+Memory management utilities for the text2sql agent.
+"""
+import gc
+import os
+import time
+import psutil
+import threading
+import weakref
+from contextlib import contextmanager
+from typing import Optional, Callable, Dict, Any
+
+class MemoryManager:
+    """Memory manager for monitoring and managing system memory."""
+    
+    def __init__(self, threshold_mb: float = 1000.0, critical_mb: float = 300.0,
+                 check_interval: float = 0.5, monitor_active: bool = True):
+        """Initialize the memory manager.
+
+        Args:
+            threshold_mb: Memory threshold in MB below which to take action
+            critical_mb: Critical memory level in MB below which to abort operations
+            check_interval: Interval in seconds for background memory checks
+            monitor_active: Whether to start the background monitor thread
+        """
+        # Allow environment overrides to tune aggressiveness
+        try:
+            _env_threshold = os.environ.get("MEM_THRESHOLD_MB")
+            _env_critical = os.environ.get("MEM_CRITICAL_MB")
+            if _env_threshold: threshold_mb = float(_env_threshold)
+            if _env_critical: critical_mb = float(_env_critical)
+        except Exception:
+            pass
+        self.threshold_mb = threshold_mb
+        self.critical_mb = critical_mb
+        self.check_interval = check_interval
+        self._lock = threading.RLock()
+        self._monitor_thread = None
+        self._keep_monitoring = False
+
+        # Track instances with weak references when possible to avoid holding
+        # strong references that prevent GC. Keep a fallback list for objects
+        # that are not weak-referenceable.
+        self._active_llm_refs = []  # list of weakref.ref where possible
+        self._active_llm_strong = []  # fallback strong refs
+        self._instance_count = 0
+
+        # Soft cap to avoid accidentally registering many parallel LLMs
+        self.max_instances = 1
+
+        # Margin (MB) above critical required to allow new registrations
+        self.registration_margin_mb = 200.0
+
+        # Track last emergency cleanup time to avoid repeated rapid-fire releases
+        self._last_emergency = 0.0
+        self._emergency_cooldown = 10.0  # seconds between emergency attempts
+        # Counter-based suspension flag to prevent emergency releases while in critical sections
+        self._suspend_emergency_depth = 0
+
+        # Minimum available memory (MB) at which we will STILL retain a single LLM instance
+        # even if below critical, to avoid thrashing (re-init cost is high). Configurable.
+        try:
+            self.min_retain_mb = float(os.environ.get("MIN_RETAIN_LLM_MB", "250"))
+        except Exception:
+            self.min_retain_mb = 250.0
+
+        # Start memory monitor if requested
+        if monitor_active:
+            self.start_monitor()
+
+    def _prune_active_instances(self):
+        """Remove dead weakrefs and return list of live instances."""
+        live = []
+        # Prune weak refs
+        for ref in list(self._active_llm_refs):
+            inst = ref()
+            if inst is None:
+                try:
+                    self._active_llm_refs.remove(ref)
+                except ValueError:
+                    pass
+            else:
+                live.append(inst)
+        # Add strong refs
+        for inst in list(self._active_llm_strong):
+            if inst is not None:
+                live.append(inst)
+        return live
+    
+    def register_llm_instance(self, llm_instance):
+        """Register an LLM instance for tracking.
+
+        This will refuse registration (and attempt to release the provided
+        instance) if the system memory is too low. It prefers weak references
+        so that the manager itself does not keep objects alive.
+        """
+        with self._lock:
+            # Prune dead weakrefs first
+            self._prune_active_instances()
+
+            stats = self.get_memory_stats()
+            # Relax policy: always allow the FIRST instance as long as we are above critical.
+            # Only enforce margin for additional instances (even though max_instances is 1 today).
+            if self._instance_count == 0:
+                if stats['available_mb'] < self.critical_mb:
+                    # Too dangerous even for first instance
+                    try:
+                        if hasattr(llm_instance, 'close'):
+                            llm_instance.close()
+                        elif hasattr(llm_instance, 'release'):
+                            llm_instance.release()
+                    except Exception:
+                        pass
+                    raise MemoryError(f"Refusing first LLM registration: below critical ({stats['available_mb']:.1f}MB < {self.critical_mb}MB)")
+            else:
+                # For subsequent instances keep the stricter margin
+                if stats['available_mb'] < (self.critical_mb + self.registration_margin_mb):
+                    try:
+                        if hasattr(llm_instance, 'close'):
+                            llm_instance.close()
+                        elif hasattr(llm_instance, 'release'):
+                            llm_instance.release()
+                    except Exception:
+                        pass
+                    raise MemoryError(f"Refusing to register additional LLM (avail={stats['available_mb']:.1f}MB; need >= {self.critical_mb + self.registration_margin_mb:.1f}MB)")
+
+            # Enforce max instances cap
+            if self._instance_count >= self.max_instances:
+                raise MemoryError(f"Refusing to register LLM instance: max_instances={self.max_instances} reached")
+
+            # Try to create a weakref to avoid holding a strong reference
+            try:
+                ref = weakref.ref(llm_instance, lambda r: None)
+                self._active_llm_refs.append(ref)
+            except TypeError:
+                # Object not weak-referenceable; store strong ref but warn
+                self._active_llm_strong.append(llm_instance)
+
+            self._instance_count += 1
+            print(f"LLM instance registered (total: {self._instance_count})")
+    
+    def unregister_llm_instance(self, llm_instance):
+        """Unregister an LLM instance from tracking."""
+        with self._lock:
+            # Remove from weak refs
+            removed = False
+            for ref in list(self._active_llm_refs):
+                inst = ref()
+                if inst is None:
+                    # Dead ref; prune
+                    try:
+                        self._active_llm_refs.remove(ref)
+                    except ValueError:
+                        pass
+                    continue
+                if inst is llm_instance:
+                    try:
+                        self._active_llm_refs.remove(ref)
+                    except ValueError:
+                        pass
+                    removed = True
+                    break
+
+            # Remove from strong refs
+            if not removed:
+                for i, instance in enumerate(list(self._active_llm_strong)):
+                    if instance is llm_instance:
+                        del self._active_llm_strong[i]
+                        removed = True
+                        break
+
+            if removed:
+                self._instance_count = max(0, self._instance_count - 1)
+                print(f"LLM instance unregistered (remaining: {self._instance_count})")
+    
+    def get_memory_stats(self) -> Dict[str, float]:
+        """Get current memory statistics."""
+        vm = psutil.virtual_memory()
+        return {
+            "total_mb": vm.total / (1024 * 1024),
+            "available_mb": vm.available / (1024 * 1024),
+            "used_mb": vm.used / (1024 * 1024),
+            "percent_used": vm.percent
+        }
+    
+    def is_memory_safe(self) -> bool:
+        """Check if memory is at a safe level."""
+        return self.get_memory_stats()["available_mb"] >= self.threshold_mb
+    
+    def is_memory_critical(self) -> bool:
+        """Check if memory is at a critical level."""
+        stats = self.get_memory_stats()
+        available = stats["available_mb"]
+        is_critical = available <= self.critical_mb
+        
+        # Debug logging for memory state
+        if is_critical:
+            print(f"🚨 [MEMORY_DEBUG] Critical state: {available:.1f}MB <= {self.critical_mb}MB threshold")
+        
+        return is_critical
+    
+    def force_collect_garbage(self):
+        """Force aggressive garbage collection."""
+        # Run multiple collection cycles
+        for i in range(3):
+            gc.collect(i)
+        
+        # Clear any reference cycles
+        if hasattr(gc, 'freeze'):
+            gc.freeze()
+        
+        # Clear module caches if possible
+        if hasattr(gc, 'clear_caches'):
+            gc.clear_caches()
+            
+        # Log memory status after cleanup
+        stats = self.get_memory_stats()
+        #print(f"After garbage collection: {stats['available_mb']:.2f}MB available ({stats['percent_used']}% used)")
+    
+    def _monitor_memory(self):
+        """Background thread for memory monitoring."""
+        while self._keep_monitoring:
+            stats = self.get_memory_stats()
+            
+            # Check for critical memory conditions
+            if stats["available_mb"] <= self.critical_mb:
+                #print(f"⚠️ CRITICAL MEMORY WARNING: Only {stats['available_mb']:.2f}MB available!")
+                self.force_collect_garbage()
+                
+                # If still critical after cleanup, take more drastic measures
+                if self.is_memory_critical():
+                    now = time.time()
+                    # Only attempt emergency release if cooldown passed
+                    if now - self._last_emergency > self._emergency_cooldown:
+                        # Check if we have tracked instances (prune dead refs first)
+                        live = self._prune_active_instances()
+                        if live:
+                            # If only a single LLM instance and we still have above min_retain_mb, skip releasing to prevent constant re-init.
+                            if len(live) == 1 and stats["available_mb"] > self.min_retain_mb:
+                                print(f"⚠️ Critical memory but retaining single LLM (avail={stats['available_mb']:.1f}MB > {self.min_retain_mb}MB safeguard)")
+                            else:
+                                # Skip emergency release while suspended
+                                if self._suspend_emergency_depth > 0:
+                                    print(f"🚫 Emergency release suppressed (depth={self._suspend_emergency_depth}) during critical section")
+                                else:
+                                    self._emergency_release_llm_instances()
+                        else:
+                            # Log once per cooldown when no instances are registered
+                            print("🚨 Emergency memory state detected but no LLM instances registered to release")
+                        self._last_emergency = now
+                    else:
+                        # Cooldown active; skip repeated emergency attempts
+                        pass
+            
+            # Regular check for low memory
+            elif stats["available_mb"] <= self.threshold_mb:
+                #print(f"⚠️ Low memory warning: {stats['available_mb']:.2f}MB available")
+                self.force_collect_garbage()
+            
+            # Sleep for the check interval
+            time.sleep(self.check_interval)
+    
+    def _emergency_release_llm_instances(self):
+        """Emergency release of LLM instances to free memory."""
+        with self._lock:
+            # Build list of live instances from weak and strong refs
+            instances = self._prune_active_instances()
+            print(f"Attempting to release {len(instances)} LLM instances")
+
+            for llm in instances:
+                try:
+                    # Try to close or release resources
+                    if hasattr(llm, 'close'):
+                        llm.close()
+                    elif hasattr(llm, 'release'):
+                        llm.release()
+                    
+                    # Set to None to break references
+                    if hasattr(llm, '_client'):
+                        llm._client = None
+                    if hasattr(llm, '_model'):
+                        llm._model = None
+                except Exception as e:
+                    print(f"Error releasing LLM instance: {str(e)}")
+            
+            # Clear our tracking lists
+            self._active_llm_refs.clear()
+            self._active_llm_strong.clear()
+            self._instance_count = 0
+
+            # Force garbage collection after cleanup and attempt to return pages
+            self.force_collect_garbage()
+            try:
+                # Best-effort working set trim to return pages to OS (Windows)
+                self.trim_working_set()
+            except Exception:
+                pass
+
+    # ---------------- Public high-level cleanup APIs -----------------
+    def release_all_llms(self):
+        """Public wrapper to release all tracked LLM instances (non-emergency manual call)."""
+        self._emergency_release_llm_instances()
+
+    def trim_working_set(self):
+        """Attempt to return unused pages to OS (best-effort, platform-specific)."""
+        try:
+            import platform, ctypes
+            if platform.system().lower() == 'windows':
+                PROCESS_SET_QUOTA = 0x0100
+                PROCESS_QUERY_INFORMATION = 0x0400
+                kernel32 = ctypes.windll.kernel32
+                psapi = ctypes.windll.psapi
+                GetCurrentProcess = kernel32.GetCurrentProcess
+                hProc = GetCurrentProcess()
+                psapi.EmptyWorkingSet(hProc)
+            else:
+                # For *nix, reading /proc/self might encourage trimming after gc
+                pass
+        except Exception as e:
+            print(f"Working set trim not supported/failed: {e}")
+
+    def perform_full_cleanup(self) -> Dict[str, float]:
+        """Comprehensive cleanup: release LLMs, GC, trim working set, return new stats."""
+        before = self.get_memory_stats()
+        self.release_all_llms()
+        self.force_collect_garbage()
+        self.trim_working_set()
+        after = self.get_memory_stats()
+        delta = after['available_mb'] - before['available_mb']
+        print(f"Post-query full cleanup reclaimed {delta:.2f}MB (avail: {after['available_mb']:.2f}MB)")
+        return after
+
+    # --------------- Critical section helpers ---------------
+    @contextmanager
+    def suspend_emergency(self, reason: str = ""):
+        """Context manager to temporarily suspend emergency LLM releases.
+
+        Use around critical sections (e.g., active LLM generation) to prevent
+        the monitor thread from tearing down the only LLM mid-call, which can
+        cause intermittent generation failures.
+        """
+        with self._lock:
+            self._suspend_emergency_depth += 1
+            if reason:
+                print(f"⏸️ Suspended emergency releases (depth={self._suspend_emergency_depth}) reason={reason}")
+        try:
+            yield
+        finally:
+            with self._lock:
+                self._suspend_emergency_depth = max(0, self._suspend_emergency_depth - 1)
+                if reason:
+                    print(f"▶️ Resumed emergency releases (depth={self._suspend_emergency_depth}) reason={reason}")
+    
+    def start_monitor(self):
+        """Start the background memory monitor."""
+        with self._lock:
+            if self._monitor_thread is None or not self._monitor_thread.is_alive():
+                self._keep_monitoring = True
+                self._monitor_thread = threading.Thread(
+                    target=self._monitor_memory,
+                    daemon=True,
+                    name="MemoryMonitorThread"
+                )
+                self._monitor_thread.start()
+                print("Memory monitor started in background")
+    
+    def stop_monitor(self):
+        """Stop the background memory monitor."""
+        with self._lock:
+            self._keep_monitoring = False
+            if self._monitor_thread and self._monitor_thread.is_alive():
+                self._monitor_thread.join(timeout=1.0)
+                print("Memory monitor stopped")
+
+# Create a global memory manager instance
+memory_manager = MemoryManager(
+    threshold_mb=500.0,   # 500MB — warning/action threshold (reduced from 1000MB)
+    critical_mb=300.0,    # 300MB — BSOD-protection hard floor (unchanged)
+    check_interval=1.0,   # Check every second
+    monitor_active=True   # Start monitoring immediately
+)
+
+class TimeoutManager:
+    """Manages operation timeouts with proper cleanup."""
+    
+    @staticmethod
+    def run_with_timeout(func: Callable, timeout: float, *args, **kwargs) -> Any:
+        """
+        Run a function with a timeout.
+        
+        Args:
+            func: Function to run
+            timeout: Timeout in seconds
+            *args, **kwargs: Arguments to pass to the function
+            
+        Returns:
+            The result of the function
+            
+        Raises:
+            TimeoutError: If the function times out
+        """
+        result = None
+        exception = None
+        completed = threading.Event()
+        
+        # Thread function
+        def worker():
+            nonlocal result, exception
+            try:
+                result = func(*args, **kwargs)
+            except Exception as e:
+                exception = e
+            finally:
+                completed.set()
+        
+        # Start the worker thread
+        thread = threading.Thread(target=worker, daemon=True)
+        thread.start()
+        
+        # Wait for completion or timeout
+        if not completed.wait(timeout):
+            # Force garbage collection before raising timeout
+            memory_manager.force_collect_garbage()
+            
+            # Raise timeout error
+            raise TimeoutError(f"Operation timed out after {timeout} seconds")
+        
+        # If there was an exception, raise it
+        if exception is not None:
+            raise exception
+        
+        return result
+
+def safe_check_memory(threshold_mb: Optional[float] = None) -> bool:
+    """
+    Safely check if there's enough memory available.
+    Also performs garbage collection if memory is low.
+    
+    Args:
+        threshold_mb: Optional threshold in MB (uses memory manager's threshold if None)
+        
+    Returns:
+        True if memory is safe, False if it's below threshold
+    """
+    # Use memory manager's threshold if none provided
+    if threshold_mb is None:
+        threshold_mb = memory_manager.threshold_mb
+    
+    # Get current memory stats
+    stats = memory_manager.get_memory_stats()
+    
+    # Log memory status
+    print(f"Memory status: {stats['available_mb']:.2f}MB available of {stats['total_mb']:.2f}MB total")
+    
+    # If memory is below threshold, run garbage collection
+    if stats['available_mb'] < threshold_mb:
+        memory_manager.force_collect_garbage()
+        
+        # Get updated stats after garbage collection
+        stats = memory_manager.get_memory_stats()
+        
+        # If still below threshold, return False
+        if stats['available_mb'] < threshold_mb:
+            print(f"⚠️ Low memory warning: Only {stats['available_mb']:.2f}MB available, need at least {threshold_mb:.2f}MB")
+            return False
+    
+    return True

Agent/merge_lora_to_ollama.py

@@ -0,0 +1,244 @@
+"""Merge a PEFT LoRA adapter (Gemma 1B) into base weights and prepare an Ollama model directory.
+
+Usage (after installing extra deps in requirements.txt):
+    python Agent/merge_lora_to_ollama.py \
+        --base google/gemma-1.1-1b-it \
+        --adapter-dir Agent/Gemma3_1B_weights \
+        --out-dir merged_gemma1b_text2sql \
+        --ollama-model-name gemma1b-text2sql
+
+Then create the Ollama model:
+    (In PowerShell)
+    ollama create gemma1b-text2sql -f merged_gemma1b_text2sql/Modelfile
+
+Finally set FINETUNED_MODEL_NAME=gemma1b-text2sql in your .env (or environment) and run the agent.
+
+Notes:
+- Ensure you used the SAME base checkpoint for finetuning as you pass via --base.
+- Script keeps everything on CPU (device_map="cpu") to stay RAM friendly; you can add --load-in-8bit if you install bitsandbytes and have GPU.
+"""
+import argparse
+import json
+import os
+import sys
+from pathlib import Path
+
+from huggingface_hub import HfApi
+try:  # Compatibility across hub versions
+    from huggingface_hub.errors import RepositoryNotFoundError  # type: ignore
+except Exception:  # Older versions
+    class RepositoryNotFoundError(Exception):
+        pass
+
+import torch
+from transformers import AutoModelForCausalLM, AutoTokenizer
+from peft import PeftModel
+
+
+def _detect_base_from_adapter(adapter_dir: str) -> str | None:
+    p = Path(adapter_dir)
+    resolved = p.resolve()
+    print(f"[auto-base] Inspecting adapter_dir: {adapter_dir} -> {resolved}")
+    if not resolved.exists():
+        print(f"[auto-base] Directory does not exist: {resolved}")
+        # Attempt smart fallbacks (common when user runs inside Agent/ already)
+        candidates = []
+        name = p.name
+        cwd = Path.cwd()
+        candidates.append(cwd / name)
+        candidates.append(cwd.parent / name)
+        candidates.append(cwd.parent / 'Agent' / name)
+        for cand in candidates:
+            if cand.exists():
+                print(f"[auto-base] Found alternative adapter dir: {cand}")
+                resolved = cand
+                break
+        if not resolved.exists():
+            print(f"[auto-base] Fallback search failed. Tried: {[str(c) for c in candidates]}")
+            return None
+    cfg_path = resolved / "adapter_config.json"
+    if not cfg_path.exists():
+        try:
+            contents = [c.name for c in resolved.iterdir()]
+        except Exception:
+            contents = []
+        print(f"[auto-base] adapter_config.json not found in {resolved}. Contents: {contents}")
+        return None
+    try:
+        raw = cfg_path.read_text(encoding="utf-8")
+        print(f"[auto-base] Found adapter_config.json ({len(raw)} bytes)")
+        data = json.loads(raw)
+        base = data.get("base_model_name_or_path")
+        print(f"[auto-base] base_model_name_or_path: {base}")
+        return base
+    except Exception as e:
+        print(f"[auto-base] Failed reading adapter_config.json: {e}")
+        return None
+
+
+def merge_adapter(base: str, adapter_dir: str, out_dir: str, ollama_model_name: str, template_temperature: float = 0.05, hf_token: str | None = None, local_files_only: bool = False, auto_base: bool = False, trust_remote_code: bool = True):
+    out_path = Path(out_dir)
+    out_path.mkdir(parents=True, exist_ok=True)
+
+    if auto_base or base.lower() == "auto":
+        detected = _detect_base_from_adapter(adapter_dir)
+        if detected:
+            print(f"[auto-base] Detected base model from adapter_config: {detected}")
+            base = detected
+        else:
+            raise RuntimeError("Could not auto-detect base model; supply --base explicitly.")
+
+    auth_kwargs = {}
+    if hf_token:
+        # Only pass 'token' (use_auth_token deprecated & causes conflict if both set)
+        auth_kwargs = {"token": hf_token}
+        os.environ.setdefault("HF_TOKEN", hf_token)
+        os.environ.setdefault("HUGGINGFACEHUB_API_TOKEN", hf_token)
+        print(f"Using HF token (masked): {hf_token[:8]}... (length {len(hf_token)})")
+
+    # Quick repo existence check (skipped if local_files_only)
+    if not local_files_only:
+        try:
+            HfApi().model_info(base, token=hf_token)
+        except RepositoryNotFoundError:
+            print(f"\n[ERROR] Base model repo '{base}' not found on Hugging Face.\n")
+            print("Troubleshooting:")
+            print("  - Verify the exact model id: e.g. 'google/gemma-2-2b-it' or 'google/gemma-1.1-2b-it'.")
+            print("  - If it's a gated/private model ensure your token has accepted the terms.")
+            print("  - Pass --hf-token YOUR_TOKEN or set HF_TOKEN env var.")
+            return 2
+        except Exception as e:
+            print(f"[WARN] Could not verify model existence pre-download: {e}")
+
+    print(f"Loading base model: {base} (local_files_only={local_files_only})")
+    try:
+        tokenizer = AutoTokenizer.from_pretrained(base, local_files_only=local_files_only, **auth_kwargs)
+    except Exception as e:
+        msg = str(e)
+        if "403" in msg or "Forbidden" in msg:
+            print("\n[AUTH ERROR] 403 while fetching tokenizer/config.")
+            print("Checklist:")
+            print("  1. Ensure token has: 'Read access to public gated models'.")
+            print("  2. Visit model page and accept license (if not already).")
+            print("  3. If using a fine-grained token, enable 'Gated repos' permission.")
+            print("  4. Retry with: --hf-token $env:HF_TOKEN (PowerShell: $env:HF_TOKEN='xxxx')")
+            print("  5. If model already cached on another machine, copy that ~/.cache/huggingface/hub directory.")
+        raise
+
+    # Keep on CPU to reduce VRAM needs; if you want faster and have GPU, remove device_map and add torch_dtype.
+    model_kwargs = dict(
+        device_map="cpu",
+        local_files_only=local_files_only,
+        torch_dtype=torch.float16 if torch.cuda.is_available() else torch.float32,
+        **auth_kwargs,
+    )
+    if trust_remote_code:
+        model_kwargs["trust_remote_code"] = True
+    # Lower RAM peak during load
+    model_kwargs["low_cpu_mem_usage"] = True
+
+    try:
+        print("[stage] Downloading / loading base model weights...")
+        base_model = AutoModelForCausalLM.from_pretrained(base, **model_kwargs)
+        print("[stage] Base model loaded successfully.")
+    except ValueError as ve:
+        msg = str(ve)
+        if "does not recognize this architecture" in msg or "model type" in msg:
+            print("[model-load] Unrecognized model type. Attempting retry after upgrading transformers may be required.")
+            print("Suggestion: pip install --upgrade transformers huggingface_hub --no-cache-dir")
+            if not trust_remote_code:
+                print("Retrying with trust_remote_code=True ...")
+                model_kwargs["trust_remote_code"] = True
+                base_model = AutoModelForCausalLM.from_pretrained(base, **model_kwargs)
+            else:
+                # Last resort: advise source install
+                raise
+        else:
+            raise
+    except Exception as e:
+        msg = str(e)
+        if "403" in msg or "Forbidden" in msg:
+            print("\n[AUTH ERROR] 403 while fetching model weights (config or pytorch model). See guidance above.")
+        raise
+
+    print(f"Loading LoRA adapter from: {adapter_dir}")
+    peft_model = PeftModel.from_pretrained(base_model, adapter_dir)
+    print("[stage] LoRA adapter loaded.")
+
+    print("Merging LoRA weights into base weights (this may take a minute)...")
+    merged_model = peft_model.merge_and_unload()
+    print("[stage] Merge complete in-memory.")
+
+    print(f"Saving merged model to: {out_dir}")
+    merged_model.save_pretrained(out_dir)
+    tokenizer.save_pretrained(out_dir)
+    print("[stage] Saved model and tokenizer.")
+
+    # Create a simple Modelfile for Ollama (avoid nested triple-quote confusion)
+    modelfile_lines = [
+        # Use current directory as model context so Modelfile works regardless of parent path
+        "FROM .",
+        f"PARAMETER temperature {template_temperature}",
+        "# Basic template; adjust if you have a custom chat format.",
+        'TEMPLATE """{{ .Prompt }}"""',
+        ""
+    ]
+    modelfile = "\n".join(modelfile_lines)
+    (out_path / "Modelfile").write_text(modelfile, encoding="utf-8")
+
+    meta = {
+        "base": base,
+        "adapter_dir": adapter_dir,
+        "merged_dir": out_dir,
+        "ollama_model_name": ollama_model_name,
+        "temperature": template_temperature,
+    }
+    (out_path / "merge_metadata.json").write_text(json.dumps(meta, indent=2), encoding="utf-8")
+
+    print("Merge complete. Next steps:")
+    print(f"  1. ollama create {ollama_model_name} -f {out_dir}/Modelfile")
+    print(f"  2. Set FINETUNED_MODEL_NAME={ollama_model_name}")
+    print("  3. Run your agent code again.")
+
+
+def main():
+    if len(sys.argv) == 1:
+        print("\n[merge_lora_to_ollama] Missing arguments. Example usage:\n")
+        print("python Agent/merge_lora_to_ollama.py \\")
+        print("  --base google/gemma-1.1-1b-it \\")
+        print("  --adapter-dir Agent/Gemma3_1B_weights \\")
+        print("  --out-dir merged_gemma1b_text2sql \\")
+        print("  --ollama-model-name gemma1b-text2sql\n")
+        return 1
+
+    parser = argparse.ArgumentParser(description="Merge a Gemma LoRA adapter for Ollama usage.")
+    parser.add_argument("--base", required=True, help="Base HF model id used during finetuning (e.g., google/gemma-3-1b-it or 'auto' to read adapter_config)")
+    parser.add_argument("--auto-base", action="store_true", help="Ignore --base value and read base from adapter_config.json")
+    parser.add_argument("--adapter-dir", required=True, help="Path to the LoRA adapter directory (with adapter_model.safetensors)")
+    parser.add_argument("--out-dir", required=True, help="Directory to write merged weights and Modelfile")
+    parser.add_argument("--ollama-model-name", required=True, help="Target Ollama model name to create")
+    parser.add_argument("--temperature", type=float, default=0.05, help="Default temperature for Modelfile")
+    parser.add_argument("--hf-token", help="Hugging Face access token (optional; else read HF_TOKEN / HUGGINGFACEHUB_API_TOKEN env)")
+    parser.add_argument("--local-files-only", action="store_true", help="Do not attempt to download; require weights to be cached locally")
+    parser.add_argument("--no-trust-remote-code", action="store_true", help="Disable trust_remote_code (not recommended for Gemma 3)")
+    args = parser.parse_args()
+
+    hf_token = args.hf_token or os.environ.get("HF_TOKEN") or os.environ.get("HUGGINGFACEHUB_API_TOKEN")
+    if not hf_token:
+        print("[INFO] No HF token provided. If the model is gated/private, pass --hf-token or set HF_TOKEN env.")
+
+    merge_adapter(
+        base=args.base,
+        adapter_dir=args.adapter_dir,
+        out_dir=args.out_dir,
+        ollama_model_name=args.ollama_model_name,
+        template_temperature=args.temperature,
+        hf_token=hf_token,
+        local_files_only=args.local_files_only,
+        auto_base=args.auto_base,
+        trust_remote_code=not args.no_trust_remote_code,
+    )
+
+
+if __name__ == "__main__":
+    main()

Agent/merge_qwen_to_ollama.py

@@ -0,0 +1,181 @@
+
+"""Merge a PEFT LoRA adapter (Qwen 2.5 3b) into base weights and prepare an Ollama model directory.
+
+This is a standalone, non-destructive helper. It follows the same pattern as
+`Agent/merge_lora_to_ollama.py` but defaults to a Qwen base and includes flags
+for optional low-memory / 8-bit loading.
+
+Usage (PowerShell):
+   Agent/merge_qwen_to_ollama.py \
+    --base Qwenpython/Qwen2.5-3B-Instruct \
+    --adapter-dir Agent/Qwen2.5_3b_weights \
+    --out-dir merged_qwen2_5_3b_text2sql \
+    --ollama-model-name qwen2.5-text2sql
+
+Notes:
+- Keeps merge on CPU to minimize peak memory.
+- If you have a GPU and want faster tokenizers/model ops, see OLLAMA GPU notes file.
+"""
+import argparse
+import json
+import os
+import sys
+from pathlib import Path
+
+try:
+    from huggingface_hub import HfApi
+except Exception:
+    HfApi = None
+
+import torch
+from transformers import AutoModelForCausalLM, AutoTokenizer
+from peft import PeftModel
+from transformers import BitsAndBytesConfig
+
+def _detect_base_from_adapter(adapter_dir: str) -> str | None:
+    p = Path(adapter_dir)
+    resolved = p.resolve()
+    print(f"[auto-base] Inspecting adapter_dir: {adapter_dir} -> {resolved}")
+    if not resolved.exists():
+        print(f"[auto-base] Directory does not exist: {resolved}")
+        # Attempt smart fallbacks (common when user runs inside Agent/ already)
+        candidates = []
+        name = p.name
+        cwd = Path.cwd()
+        candidates.append(cwd / name)
+        candidates.append(cwd.parent / name)
+        candidates.append(cwd.parent / 'Agent' / name)
+        for cand in candidates:
+            if cand.exists():
+                print(f"[auto-base] Found alternative adapter dir: {cand}")
+                resolved = cand
+                break
+        if not resolved.exists():
+            print(f"[auto-base] Fallback search failed. Tried: {[str(c) for c in candidates]}")
+            return None
+    cfg_path = resolved / "adapter_config.json"
+    if not cfg_path.exists():
+        try:
+            contents = [c.name for c in resolved.iterdir()]
+        except Exception:
+            contents = []
+        print(f"[auto-base] adapter_config.json not found in {resolved}. Contents: {contents}")
+        return None
+    try:
+        raw = cfg_path.read_text(encoding="utf-8")
+        print(f"[auto-base] Found adapter_config.json ({len(raw)} bytes)")
+        data = json.loads(raw)
+        base = data.get("base_model_name_or_path")
+        print(f"[auto-base] base_model_name_or_path: {base}")
+        return base
+    except Exception as e:
+        print(f"[auto-base] Failed reading adapter_config.json: {e}")
+        return None
+
+def merge_adapter(base: str, adapter_dir: str, out_dir: str, ollama_model_name: str, template_temperature: float = 0.05, hf_token: str | None = None, local_files_only: bool = False, trust_remote_code: bool = True, auto_base: bool = False):
+    out_path = Path(out_dir)
+    out_path.mkdir(parents=True, exist_ok=True)
+
+    # If requested, detect base from adapter_config.json
+    if auto_base or (isinstance(base, str) and base.lower() == "auto"):
+        detected = _detect_base_from_adapter(adapter_dir)
+        if detected:
+            print(f"[auto-base] Detected base model from adapter_config: {detected}")
+            base = detected
+        else:
+            raise RuntimeError("Could not auto-detect base model; supply --base explicitly.")
+
+    # Load the tokenizer first, it's small and fast
+    try:
+        tokenizer = AutoTokenizer.from_pretrained(base, local_files_only=local_files_only)
+    except Exception as e:
+        print(f"Failed loading tokenizer for base '{base}': {e}")
+        raise
+
+    # Configure 4-bit quantization with the necessary offload flag
+    quantization_config = BitsAndBytesConfig(
+        load_in_4bit=True,
+        bnb_4bit_quant_type="nf4",
+        bnb_4bit_compute_dtype=torch.float16,
+        bnb_4bit_use_double_quant=True,
+        llm_int8_enable_fp32_cpu_offload=True, # THIS IS CRUCIAL
+    )
+
+    print(f"Loading base model: {base} (using 4-bit quantization with auto device mapping) 🧠💡")
+
+    try:
+        # Use the BitsAndBytesConfig and let device_map="auto" handle the offloading
+        base_model = AutoModelForCausalLM.from_pretrained(
+            base,
+            quantization_config=quantization_config,
+            device_map="auto",
+            low_cpu_mem_usage=True,
+            local_files_only=local_files_only,
+            trust_remote_code=trust_remote_code,
+        )
+    except Exception as e:
+        print(f"Failed loading base model '{base}': {e}")
+        raise
+
+    print(f"Loading LoRA adapter from: {adapter_dir}")
+    peft_model = PeftModel.from_pretrained(base_model, adapter_dir)
+    print("LoRA adapter loaded.")
+
+    print("Merging LoRA weights into base weights (in-memory)...")
+    merged_model = peft_model.merge_and_unload()
+    print("Merge complete.")
+
+    print(f"Saving merged model to: {out_dir}")
+    merged_model.save_pretrained(out_dir)
+    tokenizer.save_pretrained(out_dir)
+
+    modelfile_lines = [
+        "FROM .",
+        f"PARAMETER temperature {template_temperature}",
+        'TEMPLATE """{{ .Prompt }}"""',
+        "",
+    ]
+    (out_path / "Modelfile").write_text("\n".join(modelfile_lines), encoding="utf-8")
+
+    meta = {
+        "base": base,
+        "adapter_dir": adapter_dir,
+        "merged_dir": out_dir,
+        "ollama_model_name": ollama_model_name,
+        "temperature": template_temperature,
+    }
+    (out_path / "merge_metadata.json").write_text(json.dumps(meta, indent=2), encoding="utf-8")
+
+    print("Merge complete. Next steps:")
+    print(f" 1. ollama create {ollama_model_name} -f {out_dir}/Modelfile")
+    print(f" 2. Set FINETUNED_MODEL_NAME={ollama_model_name} in your environment")
+    print(" 3. Run your agent code again.")
+
+def main():
+    parser = argparse.ArgumentParser(description="Merge a Qwen LoRA adapter for Ollama usage.")
+    parser.add_argument("--base", required=True, help="Base HF model id used during finetuning (default: qwen2.5:3b-instruct)")
+    parser.add_argument("--auto-base", action="store_true", help="Ignore --base value and read base from adapter_config.json")
+    parser.add_argument("--adapter-dir", required=True, help="Path to the LoRA adapter directory (with adapter_model.safetensors)")
+    parser.add_argument("--out-dir", required=True, help="Directory to write merged weights and Modelfile")
+    parser.add_argument("--ollama-model-name", required=True, help="Target Ollama model name to create")
+    parser.add_argument("--temperature", type=float, default=0.05, help="Default temperature for Modelfile")
+    parser.add_argument("--hf-token", help="Hugging Face access token (optional)")
+    parser.add_argument("--local-files-only", action="store_true", help="Do not attempt to download; require weights to be cached locally")
+    parser.add_argument("--no-trust-remote-code", action="store_true", help="Disable trust_remote_code (not recommended for some models)")
+    args = parser.parse_args()
+
+    hf_token = args.hf_token or os.environ.get("HF_TOKEN") or os.environ.get("HUGGINGFACEHUB_API_TOKEN")
+
+    merge_adapter(
+        base=args.base,
+        adapter_dir=args.adapter_dir,
+        out_dir=args.out_dir,
+        ollama_model_name=args.ollama_model_name,
+        template_temperature=args.temperature,
+        hf_token=hf_token,
+        local_files_only=args.local_files_only,
+        trust_remote_code=not args.no_trust_remote_code,
+    )
+
+if __name__ == "__main__":
+    main()

Agent/merged_model_test.py

@@ -0,0 +1,26 @@
+import json
+from safetensors.numpy import load_file
+from pathlib import Path
+
+base = Path(r"e:\Capstone_Project\text2sql-agent-mcp\merged_qwen2_5_3b_text2sql")
+safetensor = base / "model.safetensors"
+cfg_path = base / "config.json"
+
+cfg = json.loads(cfg_path.read_text(encoding="utf-8"))
+hidden = cfg.get("hidden_size")
+print("expected attn_q shape (hidden, hidden) =", hidden, hidden)
+
+print("Loading safetensors (this may use RAM)...")
+tensors = load_file(str(safetensor))
+
+# find candidate keys
+candidates = [k for k in tensors.keys() if "attn_q.weight" in k or "blk.0.attn_q.weight" in k]
+if not candidates:
+    # also show some top-level keys to help debug
+    print("No attn_q keys found. Showing first 40 keys in the safetensor:")
+    for k in list(tensors.keys())[:40]:
+        print("  ", k)
+else:
+    print("Found attn_q keys (showing up to 30):")
+    for k in candidates[:30]:
+        print(k, "->", tensors[k].shape, "dtype:", tensors[k].dtype)

Agent/migrate_to_neon.py

@@ -0,0 +1,54 @@
+import os
+import asyncio
+from sqlalchemy import create_engine, text
+from sqlalchemy.engine import URL
+import pandas as pd
+from dotenv import load_dotenv
+
+# Try to find .env in current dir and Agent dir
+load_dotenv()
+load_dotenv(dotenv_path=os.path.join(os.path.dirname(__file__), '.env'))
+load_dotenv(dotenv_path=os.path.join(os.path.dirname(__file__), '..', '.env'))
+
+# LOCAL_DB = "postgresql://postgres:postgres@localhost:5432/employee"
+LOCAL_DB = os.environ.get("DB_CONNECTION_URL")
+# NEON_DB = Your new Neon connection string
+NEON_DB = os.environ.get("NEON_DB_URL")
+
+async def migrate():
+    if not NEON_DB:
+        print("❌ Error: NEON_DB_URL not found in environment variables.")
+        return
+
+    print("🚀 Starting Cloud Migration to Neon.tech...")
+    
+    # 1. Connect to Local
+    local_engine = create_engine(LOCAL_DB)
+    
+    # 2. Connect to Neon (Ensure SSL)
+    neon_uri = NEON_DB
+    if 'sslmode=' not in neon_uri:
+        sep = '&' if '?' in neon_uri else '?'
+        neon_uri += f"{sep}sslmode=require"
+    
+    neon_engine = create_engine(neon_uri)
+
+    tables = ["department", "employee", "dept_emp", "dept_manager", "salary", "title"]
+
+    for table in tables:
+        print(f"📦 Migrating table: {table}...")
+        try:
+            # Read data from local
+            df = pd.read_sql_table(table, local_engine)
+            print(f"   - Found {len(df)} rows.")
+            
+            # Write to Neon (replace if exists)
+            df.to_sql(table, neon_engine, if_exists='replace', index=False)
+            print(f"   ✅ {table} successfully uploaded to Neon.")
+        except Exception as e:
+            print(f"   ❌ Failed to migrate {table}: {e}")
+
+    print("\n🎉 Migration Complete! Your cloud database is ready.")
+
+if __name__ == "__main__":
+    asyncio.run(migrate())

Agent/test_adapter_weights.py

@@ -0,0 +1,45 @@
+
+# Set environment variables BEFORE importing torch
+import os
+os.environ["TORCHINDUCTOR_DISABLE"] = "1"
+os.environ["TORCH_COMPILE_DISABLE"] = "1"
+# Optional: force a specific GPU (uncomment if needed)
+# os.environ["CUDA_VISIBLE_DEVICES"] = "0"
+
+from transformers import AutoModelForCausalLM, AutoTokenizer
+from peft import PeftModel
+import torch
+
+# In-code disables for torch.compile and inductor/triton
+torch._dynamo.config.suppress_errors = True
+#torch._dynamo.config.enable = False
+torch.set_float32_matmul_precision('high')
+
+#base = "google/gemma-3-1b-it"
+#adapter = "Agent/Gemma3_1B_weights"
+base = "Qwen/Qwen2.5-3B-Instruct"
+adapter = "Agent/Qwen2.5_3b_weights"
+model = AutoModelForCausalLM.from_pretrained(
+    base,
+    dtype="auto",
+    device_map=None,
+    load_in_8bit=True,
+    low_cpu_mem_usage=True,
+    local_files_only=False  # Set to False to force download if needed
+    #trust_remote_code=True
+)
+model = PeftModel.from_pretrained(model, adapter, local_files_only=True)
+model = model.to("cuda")  # Move model to GPU
+tokenizer = AutoTokenizer.from_pretrained(base)
+
+prompt = "Schema: employee(emp_no, birth_date, first_name, last_name, gender, hire_date),\n" \
+         "department(dept_no, dept_name),\n" \
+         "dept_emp(emp_no, dept_no, from_date, to_date),\n" \
+         "salary(emp_no, salary, from_date, to_date),\n" \
+         "Write PostgreSQL query for highest and lowest salary across all departments. Do not include any explanation."
+inputs = tokenizer(prompt, return_tensors="pt")
+# Move all input tensors to CUDA
+inputs = {k: v.to("cuda") for k, v in inputs.items()}
+
+output = model.generate(**inputs, max_new_tokens=1028)
+print(tokenizer.decode(output[0], skip_special_tokens=True))

Agent/test_merged_model.py

@@ -0,0 +1,19 @@
+from safetensors.numpy import load_file
+from pathlib import Path
+
+base = Path(r"e:\Capstone_Project\text2sql-agent-mcp\merged_qwen2_5_3b_text2sql")
+safetensor = base / "model.safetensors"
+
+print("Loading safetensors (may use RAM)...")
+d = load_file(str(safetensor))
+
+weights = [k for k in d.keys() if k.endswith(".weight")]
+print("weight keys count:", len(weights))
+
+if weights:
+    for k in weights[:200]:
+        print(k, d[k].shape, d[k].dtype)
+else:
+    print("\nNo '.weight' keys found — showing first 120 keys:")
+    for k in list(d.keys())[:120]:
+        print(k)

Agent/test_query_runner.py

@@ -0,0 +1,20 @@
+import asyncio
+import os
+import sys
+# Ensure Agent package path is importable
+sys.path.insert(0, r"e:/Capstone_Project/text2sql-agent-mcp/Agent")
+import backups.agentic_workflow4 as aw
+
+async def run_one():
+    # Initialize DB (use the module's configured DB_CONNECTION_URL when available)
+    db_uri = os.environ.get('DB_CONNECTION_URL') or getattr(aw, 'DB_CONNECTION_URL', None)
+    await aw.initialize_database(db_uri)
+    # Run a single question
+    # Prefix with 're-execute' to force regeneration and bypass any cached error in memory
+    res = await aw.agentic_query_process('re-execute Who are the top 10 highest-paid employees?')
+    print('\n--- AGENT OUTPUT START ---')
+    print(res)
+    print('--- AGENT OUTPUT END ---')
+
+if __name__ == '__main__':
+    asyncio.run(run_one())

Agent/tools/document_scanner.py

@@ -0,0 +1,223 @@
+import os
+try:
+    # Ensure local .env values (OLLAMA_*) are loaded when this module is imported
+    from dotenv import load_dotenv
+    load_dotenv()
+except Exception:
+    pass
+import uuid
+import re
+import base64
+from typing import Dict, Any, Optional
+
+def _mock_result(received: Optional[Dict[str, Any]] = None) -> Dict[str, Any]:
+    out = {"doc_id": str(uuid.uuid4()), "text": "(mock OCR output)", "metadata": {"pages": 1, "confidence": 0.0}}
+    if received:
+        out["received_args"] = received
+    return out
+
+
+def clean_ocr_text(raw: Optional[str]) -> str:
+    """Lightweight post-processing for OCR output to improve readability.
+
+    - turns literal "\\n" sequences into real newlines
+    - unescapes common unicode/HTML escapes when safe
+    - collapses repeated spaces and blank lines
+    - trims and merges lines that were broken mid-sentence
+    """
+    if not raw:
+        return ""
+    s = raw
+    try:
+        # Convert literal backslash-n to actual newline
+        s = s.replace('\\n', '\n')
+    except Exception:
+        pass
+
+    # Try to interpret common escape sequences (conservative)
+    try:
+        s_decoded = s.encode('utf-8', errors='surrogatepass').decode('unicode_escape')
+        # Accept decoded result if it reduces backslash artifacts
+        if '\\n' not in s_decoded:
+            s = s_decoded
+    except Exception:
+        pass
+
+    # Normalize line endings and whitespace
+    s = s.replace('\r\n', '\n').replace('\r', '\n')
+    s = re.sub(r'[ \t]{2,}', ' ', s)
+    # Collapse 3+ newlines into max 2
+    s = re.sub(r'\n{3,}', '\n\n', s)
+
+    # Trim whitespace on each line
+    lines = [ln.strip() for ln in s.splitlines()]
+
+    # Merge lines that likely were broken mid-sentence: if a line doesn't end with
+    # sentence punctuation and the next line begins with a lowercase letter or digit,
+    # join them with a space.
+    merged = []
+    i = 0
+    while i < len(lines):
+        line = lines[i]
+        if i < len(lines) - 1:
+            nxt = lines[i + 1]
+            if line and nxt and not re.search(r'[\.\!\?\:;]$', line) and re.match(r'^[a-z0-9]', nxt):
+                line = line + ' ' + nxt
+                i += 2
+                # Continue merging subsequent similar lines
+                while i < len(lines) and lines[i] and not re.search(r'[\.\!\?\:;]$', line) and re.match(r'^[a-z0-9]', lines[i]):
+                    line = line + ' ' + lines[i]
+                    i += 1
+                merged.append(line)
+                continue
+        merged.append(line)
+        i += 1
+
+    s = '\n'.join([ln for ln in merged if ln])
+
+    # Remove long runs of underscores or pipes
+    s = re.sub(r'[_\|]{2,}', '', s)
+
+    # Strip control chars except common whitespace
+    s = ''.join(ch for ch in s if ord(ch) >= 9 and ord(ch) != 11 and ord(ch) != 12)
+
+    try:
+        import html
+
+        s = html.unescape(s)
+    except Exception:
+        pass
+
+    # Decode literal unicode escapes like \u0027 -> ' and \xA4 -> currency symbol
+    try:
+        def _u_decode(m):
+            code = m.group(1)
+            try:
+                return chr(int(code, 16))
+            except Exception:
+                return m.group(0)
+
+        s = re.sub(r'\\u([0-9A-Fa-f]{4})', _u_decode, s)
+        s = re.sub(r'\\x([0-9A-Fa-f]{2})', lambda m: chr(int(m.group(1), 16)), s)
+    except Exception:
+        pass
+
+    # Remove stray leading 'n' characters that appear before uppercase words (common OCR/newline artifact)
+    try:
+        s = re.sub(r'(?<=\s|^)[nN](?=[A-Z])', '', s)
+    except Exception:
+        pass
+
+    # Ensure space after commas when missing
+    try:
+        s = re.sub(r',(?=[^\s])', ', ', s)
+    except Exception:
+        pass
+
+    # Remove spaces before punctuation
+    try:
+        s = re.sub(r'\s+([,\.\:\;\%])', r'\1', s)
+    except Exception:
+        pass
+
+    # Attempt to fix common mojibake (double-encoded UTF-8) conservatively.
+    # Only apply if there are visible 'Ã' sequences suggesting encoding issues.
+    try:
+        if 'Ã' in s or 'Â' in s:
+            try:
+                candidate = s.encode('latin-1').decode('utf-8')
+                # Accept candidate if it reduces suspicious sequences
+                if candidate.count('Ã') < s.count('Ã'):
+                    s = candidate
+            except Exception:
+                pass
+    except Exception:
+        pass
+
+    return s.strip()
+
+
+def process_image(image_url: Optional[str] = None, image_bytes: Optional[bytes] = None, options: Optional[Dict[str, Any]] = None) -> Dict[str, Any]:
+    """Process an image using local OCR first (pytesseract), with a light transformers fallback.
+
+    Accepts either `image_url` or `image_bytes` (bytes or base64 string). Returns dict: {doc_id, text, metadata}
+    """
+
+    # Lightweight imports that may be missing in minimal environments
+    try:
+        from PIL import Image
+        from io import BytesIO
+        import requests
+    except Exception:
+        # Missing imaging/network deps -> return mock with echo of args
+        return _mock_result(received={"image_url": image_url, "has_bytes": bool(image_bytes)})
+
+    # Normalize image_bytes if base64 string was passed
+    if isinstance(image_bytes, str):
+        try:
+            image_bytes = base64.b64decode(image_bytes)
+        except Exception:
+            image_bytes = None
+
+    # Acquire PIL image
+    img = None
+    try:
+        if image_bytes:
+            img = Image.open(BytesIO(image_bytes)).convert("RGB")
+        elif image_url:
+            resp = requests.get(image_url, timeout=15)
+            resp.raise_for_status()
+            img = Image.open(BytesIO(resp.content)).convert("RGB")
+        else:
+            return _mock_result(received={"image_url": image_url, "has_bytes": False})
+    except Exception:
+        return _mock_result(received={"image_url": image_url, "has_bytes": bool(image_bytes)})
+
+    # PRIORITY: Try local Tesseract (pytesseract) first — lightweight and deterministic on Windows
+
+    try:
+        import pytesseract
+        # Robustly find tesseract executable: prefer PATH, then common Windows install locations
+        try:
+            import shutil
+            found = shutil.which('tesseract')
+        except Exception:
+            found = None
+        if not found:
+            # common install locations on Windows
+            for candidate in (r"C:\Program Files\Tesseract-OCR\tesseract.exe", r"C:\Program Files (x86)\Tesseract-OCR\tesseract.exe"):
+                try:
+                    if os.path.exists(candidate):
+                        found = candidate
+                        break
+                except Exception:
+                    continue
+        try:
+            if found:
+                pytesseract.pytesseract.tesseract_cmd = found
+                print(f"[document_scanner] using tesseract executable: {found}")
+            else:
+                # leave default; pytesseract will raise a helpful error which we catch below
+                print("[document_scanner] tesseract executable not found in PATH or common locations")
+        except Exception:
+            # ignore assignment errors and let pytesseract raise on use
+            pass
+
+        try:
+            if img is not None:
+                print("[document_scanner] attempting PRIMARY pytesseract OCR")
+                ttxt = pytesseract.image_to_string(img)
+                if ttxt and ttxt.strip():
+                    print(f"[document_scanner] pytesseract returned text (len={len(ttxt.strip())})")
+                    return {"doc_id": str(uuid.uuid4()), "text": clean_ocr_text(ttxt), "metadata": {"pages": 1, "confidence": None}}
+                else:
+                    print("[document_scanner] pytesseract returned no text; falling through to model-based fallbacks")
+        except Exception as e_pyt_first:
+            print(f"[document_scanner] primary pytesseract attempt failed: {e_pyt_first}")
+    except Exception:
+        # pytesseract not installed in environment — continue to available backends
+        pass
+
+    # Last resort: return a safe mock that echoes inputs
+    # (Tesseract failed or is not installed — Q&A will still work via Granite Vision if image is saved)
+    return _mock_result(received={"image_url": image_url, "has_bytes": bool(image_bytes)})

Agent/tools/granite_vision.py

@@ -0,0 +1,120 @@
+"""
+Vision MCP tool - visual Q&A via Groq Vision API (llama-3.2-11b-vision-preview).
+
+Cloud path  : image -> Groq Llama 3.2 Vision -> structured answer (FREE, 14400 req/day)
+Local path  : image -> Ollama qwen2-vl -> answer (dev only)
+
+Registered in mcp_server.py as 'granite_vision.qa'.
+"""
+import os
+import base64
+import requests
+from typing import Optional
+
+
+def qa(image_path: str, question: str, model_path: Optional[str] = None) -> dict:
+    """Answer a question about a document image using Groq Vision or local Ollama."""
+
+    print(f"[vision.qa] image={image_path} question={question[:80]}")
+
+    # CLOUD MODE: Groq Vision API
+    if os.environ.get("USE_HF_CLOUD", "0") in ("1", "true", "True"):
+        groq_key = os.environ.get("GROQ_API_KEY")
+        groq_model = os.environ.get("GROQ_VISION_MODEL", "meta-llama/llama-4-scout-17b-16e-instruct")
+
+        if groq_key:
+            print(f"[granite_vision.qa] CLOUD MODE: {groq_model} via Groq")
+            try:
+                from groq import Groq
+
+                client = Groq(api_key=groq_key)
+
+                with open(image_path, "rb") as f:
+                    img_b64 = base64.b64encode(f.read()).decode("utf-8")
+
+                ext = os.path.splitext(image_path)[-1].lower().lstrip(".")
+                mime = "image/jpeg" if ext in ("jpg", "jpeg") else f"image/{ext}"
+
+                response = client.chat.completions.create(
+                    model=groq_model,
+                    messages=[
+                        {
+                            "role": "user",
+                            "content": [
+                                {
+                                    "type": "text",
+                                    "text": (
+                                        "You are analyzing a document image. "
+                                        "Look carefully at the layout, tables, numbers, and text. "
+                                        "Answer the following question based only on what you see:\n\n"
+                                        + question
+                                    )
+                                },
+                                {
+                                    "type": "image_url",
+                                    "image_url": {
+                                        "url": f"data:{mime};base64,{img_b64}"
+                                    }
+                                }
+                            ]
+                        }
+                    ],
+                    max_tokens=512
+                )
+
+                answer = response.choices[0].message.content.strip()
+                print(f"[granite_vision.qa] Groq answer ({len(answer)} chars): {answer[:200]}")
+                return {"answer": answer, "model": f"{groq_model} (Groq Vision)"}
+
+            except Exception as e:
+                print(f"[granite_vision.qa] Groq failed: {e}")
+                return {
+                    "answer": f"Groq Vision error: {e}",
+                    "model": groq_model,
+                    "error": str(e)
+                }
+        else:
+            print("[granite_vision.qa] GROQ_API_KEY not set in HF secrets.")
+            return {
+                "answer": "GROQ_API_KEY is not configured. Please add it in HuggingFace Space secrets.",
+                "model": "none",
+                "error": "missing_groq_key"
+            }
+
+    # LOCAL MODE (dev only): Ollama
+    local_model = os.environ.get("GRANITE_MODEL_PATH", "qwen2-vl:2b")
+    print(f"[granite_vision.qa] LOCAL MODE: sending image to Ollama ({local_model})")
+
+    try:
+        with open(image_path, "rb") as f:
+            img_b64 = base64.b64encode(f.read()).decode("utf-8")
+
+        base_url = os.environ.get("OLLAMA_BASE_URL", "http://localhost:11434").rstrip("/")
+        url = f"{base_url}/api/chat"
+        payload = {
+            "model": local_model,
+            "messages": [
+                {
+                    "role": "user",
+                    "content": f"Carefully analyze this document image. {question}",
+                    "images": [img_b64]
+                }
+            ],
+            "stream": False,
+            "keep_alive": 60,
+            "options": {"num_ctx": 1024, "temperature": 0.1}
+        }
+        r = requests.post(url, json=payload, timeout=300)
+        if r.status_code == 200:
+            answer = r.json().get("message", {}).get("content", "").strip()
+            print(f"[granite_vision.qa] Ollama answer ({len(answer)} chars): {answer[:200]}")
+            return {"answer": answer, "model": f"{local_model} (Ollama)"}
+        else:
+            msg = f"Ollama HTTP {r.status_code}: {r.text}"
+            print(f"[granite_vision.qa] {msg}")
+            return {"answer": msg, "model": f"{local_model} (Ollama)", "error": "http_error"}
+
+    except Exception as e:
+        msg = f"[ERROR] Ollama Q&A failed: {e}"
+        print(f"[granite_vision.qa] {msg}")
+        return {"answer": msg, "model": f"{local_model} (Ollama)", "error": str(e)}

Agent/worker.py

@@ -0,0 +1,235 @@
+import os
+import asyncio
+import io
+import sys
+import contextlib
+from fastapi import FastAPI, HTTPException
+from pydantic import BaseModel
+from typing import Optional
+from fastapi import File, UploadFile
+import base64
+import uuid
+import uvicorn
+
+# Worker that imports the agent and exposes a simple HTTP API
+app = FastAPI(title="Agent Worker")
+
+class QueryReq(BaseModel):
+    question: str
+    doc_id: Optional[str] = None
+    mode: Optional[str] = None  # 'text2sql' or 'ocr_qa'
+
+
+class UploadResp(BaseModel):
+    doc_id: str
+    text_preview: str
+    image_url: str
+
+class OcrQAReq(BaseModel):
+    question: str
+    doc_id: str
+
+class OcrQAResp(BaseModel):
+    doc_id: str
+    answer: str
+    used_llm: bool
+
+# Lazy import agent module (do at startup so heavy init happens in worker)
+agent = None
+# Lock to prevent concurrent capture of stdout/stderr which would interleave logs
+capture_lock = asyncio.Lock()
+
+@app.on_event("startup")
+async def startup_event():
+    global agent
+    # Import here so the LLM and DB init occur inside the worker process
+    # Use package-relative import so we load Agent.agentic_workflow (not a top-level file)
+    try:
+        from . import agentic_workflow as agentic_workflow
+    except Exception:
+        # fallback to absolute package import
+        import Agent.agentic_workflow as agentic_workflow
+    agent = agentic_workflow
+    # Initialize DB (best-effort)
+    db_url = os.environ.get("DB_CONNECTION_URL")
+    if db_url:
+        try:
+            await agent.initialize_database(uri=db_url)
+        except Exception as e:
+            # Log but continue; agentic_workflow has fallback behavior
+            print(f"Worker: initialize_database failed: {e}")
+    else:
+        print("Worker: DB_CONNECTION_URL not set; skipping DB init")
+
+@app.get("/health")
+async def health():
+    return {"status": "ok"}
+
+
+@app.post("/query")
+async def query(req: QueryReq):
+    global agent
+    if agent is None:
+        raise HTTPException(status_code=503, detail="Agent not initialized")
+    q = req.question
+    doc_id = getattr(req, 'doc_id', None)
+    mode = getattr(req, 'mode', None)
+    # Capture stdout/stderr produced by the agent for this request
+    buf_out = io.StringIO()
+    buf_err = io.StringIO()
+    async with capture_lock:
+        try:
+            with contextlib.redirect_stdout(buf_out), contextlib.redirect_stderr(buf_err):
+                # Robust mode switch: if mode is 'ocr_qa' or doc_id is present, use OCR agent; else use SQL logic
+                if (mode == 'ocr_qa') or (doc_id and (mode is None or mode == 'auto')):
+                    # Use OCR Q&A agent
+                    # ocr_agent_qa may be sync, so run in executor
+                    import asyncio
+                    answer = await asyncio.get_event_loop().run_in_executor(
+                        None, lambda: agent.ocr_agent_qa(q, doc_id)
+                    )
+                    result = answer
+                    sql_out = None  # No SQL for OCR Q&A
+                else:
+                    # Use standard Text-to-SQL agentic workflow
+                    result = await agent.agentic_query_process(q, None)
+                    try:
+                        sql_out = getattr(agent, 'LAST_GENERATED_SQL', None)
+                    except Exception:
+                        sql_out = None
+        except ValueError as ve:
+            # Treat known generation failures as normal answers so UI doesn't show nested JSON error
+            err_msg = str(ve)
+            if "Unable to generate SQL query" in err_msg:
+                result = err_msg  # Provide user-visible plain text
+                sql_out = None
+            else:
+                import traceback
+                traceback.print_exc(file=buf_err)
+                raise HTTPException(status_code=500, detail=err_msg)
+        except Exception as e:
+            import traceback
+            traceback.print_exc(file=buf_err)
+            raise HTTPException(status_code=500, detail=str(e))
+    logs = buf_out.getvalue()
+    errlogs = buf_err.getvalue()
+    if errlogs:
+        logs = logs + "\n" + errlogs
+    return {"answer": result, "logs": logs, "sql": sql_out}
+
+
+@app.post("/ocr-qa", response_model=OcrQAResp)
+async def ocr_qa(req: OcrQAReq):
+    """Answer a question about a previously uploaded document using its doc_id.
+
+    This uses the agent's ocr_agent_qa helper which pulls stored OCR text and
+    consults the LLM (or returns a low-memory fallback excerpt).
+    """
+    global agent
+    if agent is None:
+        raise HTTPException(status_code=503, detail="Agent not initialized")
+    question = req.question
+    doc_id = req.doc_id
+    # Run potentially blocking OCR QA in thread pool if it's sync
+    try:
+        answer = await asyncio.get_event_loop().run_in_executor(
+            None, lambda: agent.ocr_agent_qa(question, doc_id)
+        )
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=f"ocr_qa failed: {e}")
+    used_llm = not answer.startswith("[Low memory]") and not answer.startswith("No OCR text") and not answer.startswith("Error while")
+    return OcrQAResp(doc_id=doc_id, answer=answer, used_llm=used_llm)
+
+
+@app.post("/upload-image", response_model=UploadResp)
+async def upload_image(file: UploadFile = File(...)):
+    """Accept an image, save locally, call MCP document_scanner.process eagerly,
+    persist OCR to private chat store, and return a doc_id plus preview.
+    """
+    global agent
+    if agent is None:
+        raise HTTPException(status_code=503, detail="Agent not initialized")
+
+    # Ensure uploads directory exists
+    uploads_dir = "uploads"
+    os.makedirs(uploads_dir, exist_ok=True)
+
+    # Save uploaded file
+    filename = f"{uuid.uuid4()}_{file.filename}"
+    path = os.path.join(uploads_dir, filename)
+    try:
+        contents = await file.read()
+        with open(path, "wb") as f:
+            f.write(contents)
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=f"Failed to save upload: {e}")
+
+    # Call MCP document_scanner tool eagerly with base64 payload to avoid exposing file server
+    b64 = base64.b64encode(contents).decode('utf-8')
+    try:
+        from .mcp_client import mcp_call_tool
+        ctx = {"tool_call": {"name": "document_scanner.process", "args": {"image_bytes": b64}}}
+        resp = mcp_call_tool(prompt="run document scanner", timeout=60.0, context=ctx)
+        # Expect tool_result in response
+        tool_result = resp.get("tool_result") if isinstance(resp, dict) else None
+    except Exception:
+        # Fall back to local direct call if MCP unavailable or tool failed
+        tool_result = None
+        try:
+            if agent and hasattr(agent, 'tools') and hasattr(agent.tools, 'document_scanner'):
+                try:
+                    tool_result = agent.tools.document_scanner.process_image(image_bytes=b64)
+                except Exception:
+                    tool_result = None
+        except Exception:
+            tool_result = None
+        if not tool_result:
+            tool_result = {"doc_id": str(uuid.uuid4()), "text": "(ocr unavailable)", "metadata": {}}
+
+    # Persist OCR to private chat store file for retrieval by doc_id
+    doc_id = tool_result.get("doc_id") if isinstance(tool_result, dict) and tool_result.get("doc_id") else str(uuid.uuid4())
+    ocr_text = tool_result.get("text", "") if isinstance(tool_result, dict) else str(tool_result)
+
+    # Save to docs/ and also to chat_store_private for history
+    docs_dir = os.path.join("chat_store", "docs")
+    os.makedirs(docs_dir, exist_ok=True)
+    doc_path = os.path.join(docs_dir, f"{doc_id}.txt")
+    try:
+        with open(doc_path, "w", encoding="utf-8") as df:
+            df.write(ocr_text)
+    except Exception as e:
+        print(f"Failed to persist OCR text: {e}")
+
+    # Also persist the original image path so Granite Vision can do direct visual Q&A
+    img_path_file = os.path.join(docs_dir, f"{doc_id}.img_path")
+    try:
+        abs_img_path = os.path.abspath(path)
+        with open(img_path_file, "w", encoding="utf-8") as ipf:
+            ipf.write(abs_img_path)
+        print(f"[upload] Saved image path for doc_id={doc_id}: {abs_img_path}")
+    except Exception as e:
+        print(f"[upload] Failed to persist image path: {e}")
+
+    # Also add a short entry to chat_store_private for traceability
+    try:
+        try:
+            from .agentic_workflow import chat_store_private, ChatMessage, MessageRole
+        except ImportError:
+            from Agent.agentic_workflow import chat_store_private, ChatMessage, MessageRole
+        # Create minimal messages
+        user_msg = ChatMessage(role=MessageRole.USER, content=f"Uploaded document {doc_id}")
+        assistant_msg = ChatMessage(role=MessageRole.ASSISTANT, content=f"OCR stored: {ocr_text[:400]}")
+        chat_store_private.add_message(key="conversation", message=user_msg)
+        chat_store_private.add_message(key="conversation", message=assistant_msg)
+        chat_store_private.persist(str("chat_store/chat_store_private.json"))
+    except Exception:
+        pass
+
+    # Return doc id and small preview
+    preview = ocr_text[:400]
+    image_url = path
+    return UploadResp(doc_id=doc_id, text_preview=preview, image_url=image_url)
+
+if __name__ == '__main__':
+    port = int(os.environ.get("WORKER_PORT", 8700))
+    uvicorn.run(app, host="127.0.0.1", port=port)

Dockerfile

@@ -0,0 +1,65 @@
+# Dockerfile for Hugging Face Spaces
+FROM python:3.11-slim
+
+# Prevent Python from writing pyc files and buffering stdout/stderr
+ENV PYTHONDONTWRITEBYTECODE=1
+ENV PYTHONUNBUFFERED=1
+ENV USE_HF_CLOUD=1
+
+# Pre-set Ollama model directory so the build-time pull goes to the right place
+ENV OLLAMA_MODELS=/app/ollama_models
+ENV HOME=/app
+
+# Install system dependencies
+RUN apt-get update && apt-get install -y \
+    curl \
+    zstd \
+    tesseract-ocr \
+    libtesseract-dev \
+    gcc \
+    libpq-dev \
+    && rm -rf /var/lib/apt/lists/*
+
+# Install latest Ollama via official install script
+# (replaces the pinned v0.5.11 which cannot pull qwen3.5:2b)
+RUN curl -fsSL https://ollama.ai/install.sh | sh
+
+# Pre-bake qwen3.5:2b vision model into the image at build time.
+# The model will be stored in /app/ollama_models and available
+# immediately when the container starts — no runtime pull needed.
+RUN mkdir -p /app/ollama_models && \
+    OLLAMA_MODELS=/app/ollama_models ollama serve > /tmp/ollama_build.log 2>&1 & \
+    OLLAMA_PID=$! && \
+    echo "[BUILD] Waiting for Ollama to start (PID=$OLLAMA_PID)..." && \
+    for i in $(seq 1 30); do \
+        if curl -s http://localhost:11434/api/tags > /dev/null 2>&1; then \
+            echo "[BUILD] Ollama is ready."; break; \
+        fi; \
+        echo "[BUILD] Waiting... ($i/30)"; \
+        sleep 2; \
+    done && \
+    echo "[BUILD] Pulling qwen3.5:2b into image layers..." && \
+    OLLAMA_MODELS=/app/ollama_models ollama pull qwen3.5:2b && \
+    echo "[BUILD] Model pull complete. Stopping build-time Ollama..." && \
+    kill $OLLAMA_PID 2>/dev/null || true && \
+    wait $OLLAMA_PID 2>/dev/null || true && \
+    echo "[BUILD] qwen3.5:2b pre-baked successfully."
+
+WORKDIR /app
+
+# Install Python dependencies
+COPY requirements.txt .
+RUN pip install --no-cache-dir --upgrade pip && \
+    pip install --no-cache-dir -r requirements.txt && \
+    pip install --no-cache-dir llama-index-llms-huggingface-api huggingface_hub
+
+# Copy application code
+COPY . .
+
+# Hugging Face Spaces requires port 7860
+EXPOSE 7860
+
+# Fix line endings and permissions on startup script
+RUN sed -i 's/\r$//' start_hf.sh && chmod +x start_hf.sh
+
+CMD ["./start_hf.sh"]

README.md

@@ -0,0 +1,548 @@
+---
+title: QueryLens Text2SQL-DocVQA
+emoji: 🤖
+colorFrom: blue
+colorTo: indigo
+sdk: docker
+pinned: false
+app_port: 7860
+tags:
+  - agents
+  - mcp
+---
+
+# 🤖 Agentic Text2SQL & Document VQA
+ Multi-Agent System
+
+![Architecture](https://img.shields.io/badge/Architecture-Multi--Agent-blue)
+![Database](https://img.shields.io/badge/Database-Agnostic-success)
+![LLM](https://img.shields.io/badge/LLMs-Qwen%20%7C%20Ollama-orange)
+![Protocol](https://img.shields.io/badge/Protocol-MCP-purple)
+
+An advanced, hardware-optimized AI pipeline built on a multi-agent orchestration architecture. This system seamlessly integrates natural language Text-to-SQL generation with a Vision-Language Model (VLM) for Optical Character Recognition (OCR) and Visual Question Answering (VQA).
+
+By leveraging the Model Context Protocol (MCP), local LLMs via Ollama, and intelligent memory management, this project achieves production-grade data extraction and database querying running entirely on local hardware within tight memory constraints.
+
+---
+
+## Key Features
+
+- **Multi-Agent Orchestration:** A dynamic routing system featuring specialized agents — MemoryAgent, QueryCraftAgent, and ResultPresenterAgent — each with a clearly defined role in the pipeline.
+- **Vision-Language Question Answering:** Integrates Qwen 3.5 2B Vision for accurate data extraction from complex documents such as invoices and unstructured tables, served via Ollama.
+- **Three-Level Text-to-SQL Engine:** Translates natural language questions into syntactically valid SQL through a progressive reasoning architecture, with automatic self-correction at each escalation level.
+- **Semantic Gatekeeping:** A multi-layer validation pipeline filters hallucinated SQL, undefined aliases, missing joins, and incorrect aggregation patterns before any query reaches the database.
+- **Dynamic VRAM Orchestration:** Custom memory management algorithms automatically load and unload models from GPU VRAM to prevent out-of-memory crashes, enabling concurrent AI workloads on consumer hardware.
+- **Unified MCP Integration:** All external data sources — relational databases and vision models alike — are accessed through a single, uniform Model Context Protocol interface.
+- **Database Agnostic Design:** Architected to interface with PostgreSQL, MySQL, SQLite, MS SQL Server, and Oracle via SQLAlchemy.
+- **Conversational Interface:** A full Web UI supporting natural language interaction with persistent session context.
+
+---
+
+## System Architecture and Agentic Workflow
+
+The project employs a decoupled, asynchronous microservices architecture. The Web UI communicates with a backend worker, which orchestrates all agent requests through the Model Context Protocol.
+
+### The Three-Agent Pipeline
+
+**1. MemoryAgent**
+
+The MemoryAgent acts as the entry point for all user queries. It performs three functions before any SQL generation occurs: checking whether the current query can be satisfied from historical conversation context, rephrasing ambiguous or under-specified queries based on prior interactions, and routing validated new queries to the appropriate downstream agent. This layer prevents redundant computation and improves response latency for repeated or related questions.
+
+**2. QueryCraftAgent — The Three-Level Text-to-SQL Engine**
+
+The QueryCraftAgent is the core reasoning engine of the system. It implements a progressive escalation architecture that balances inference speed with accuracy:
+
+- **Level 1 — Pure LLM Intelligence:** Zero-shot semantic translation of natural language to SQL using Qwen 2.5 Coder. The generated query passes through a semantic validation pipeline that checks alias correctness, required table joins, aggregation integrity, and department or entity filters before being accepted.
+
+- **Level 2 — LLM with SQL Fixer:** When Level 1 fails validation, the agent enters an autonomous self-correction loop. The SQL Fixer applies a deterministic set of targeted transformations — normalizing aliases, injecting missing to_date filters, repairing phantom column references, and enforcing PostgreSQL-compliant GROUP BY clauses — before re-validating the output.
+
+- **Level 3 — LLM with MCP Schema-Grounded SQL Generation:** The deepest reasoning level, reserved for queries that fail both prior levels. The agent calls one of fourteen registered `fast_sql` MCP tools, each of which queries the live database schema via SQLAlchemy's inspector to retrieve actual column names and table structures, then programmatically constructs a valid SQL template grounded in the real schema. This template is passed as a verified hint to the LLM, which refines it before the Fixer loop runs. Because the hint is built from the actual schema rather than the LLM's training data, this level eliminates the class of failures caused by hallucinated column names or missing joins.
+
+**3. ResultPresenterAgent**
+
+The ResultPresenterAgent executes the validated SQL query against the target database, translates raw result sets into natural conversational responses, handles execution errors and edge cases, and persists the interaction to the conversation history for future MemoryAgent lookups.
+
+### Multi-Agent Orchestration Flow
+
+```mermaid
+graph TD
+    User([User]) --> Web["Web Application\nPort 8000"]
+
+    Web -->|Image Upload| Worker["Backend Worker\nPort 8700"]
+    Web -->|Text Query| Worker
+
+    subgraph MCP["MCP Tool Server — Port 8701"]
+        DocScan["document_scanner.process\nTesseract OCR"]
+        FastSQL["fast_sql tools\nSchema-Grounded SQL Generation"]
+        VisionQA["granite_vision.qa\nQwen 3.5 2B Vision"]
+    end
+
+    Worker -->|"Upload: MCP tool call"| DocScan
+    DocScan -->|"doc_id + OCR text + image path"| Store[("Document Store\nchat_store/docs/")]
+
+    Worker -->|"OCR Q&A mode\ndoc_id present"| VisionQA
+    Store -->|image path| VisionQA
+    VisionQA --> OllamaV(["Ollama\nQwen 3.5 2B Vision"])
+    OllamaV --> Presenter
+
+    Worker -->|Text Query| Memory["MemoryAgent"]
+    Memory -->|Cached answer| Presenter["ResultPresenterAgent"]
+    Memory -->|New query| QCAgent["QueryCraftAgent"]
+
+    QCAgent -->|"Level 1 and 2\nDirect Ollama call"| OllamaSQL(["Ollama\nQwen 2.5 Coder"])
+    QCAgent -->|"Level 3\nMCP tool call"| FastSQL
+    FastSQL -->|"Schema-validated SQL hint"| OllamaSQL
+    OllamaSQL -->|Raw SQL| Fixer["SQL Fixer +\nSemantic Validator"]
+    Fixer -->|"Validated SQL\nvia SQLAlchemy"| DB[("PostgreSQL / MySQL")]
+
+    DB --> Presenter
+    Presenter -->|Persist to history| Memory
+    Presenter --> Answer(["Natural Language Answer"])
+```
+
+---
+
+## Infrastructure and Deployment Architecture
+
+### Containerized Microservices
+
+The system is implemented as a Dockerized microservices architecture, with each component running as an isolated, independently deployable service. Docker Compose orchestrates three services: the main application and agent worker, the MCP tool server, and the relational databases (PostgreSQL and MySQL). Database containers are provisioned with health checks using `pg_isready` and `mysqladmin ping`, and the application container is configured with `condition: service_healthy` dependencies to eliminate the startup race condition inherent in naive `depends_on` configurations.
+
+### Native-Bridge Hybrid Offloading
+
+For peak inference performance, the system uses a hybrid offloading strategy. The application logic and MCP servers run inside Docker containers for portability and isolation, while Ollama and all GPU inference are offloaded to the native Windows host and accessed by the containers over the WSL2 host bridge network (`OLLAMA_BASE_URL=http://<wsl2-host-ip>:11434`).
+
+This architecture eliminates the virtualization overhead introduced by Docker Desktop's Linux VM layer. By giving the inference engine direct access to NVIDIA CUDA cores and the full system memory pool — rather than a virtualized slice — the 1.5B parameter model achieves response times measured in seconds rather than minutes. The separation also means that GPU memory pressure from Ollama does not compete with the container's allocated memory ceiling.
+
+### Database Health and Startup Reliability
+
+A production-grade startup sequence is enforced through Docker health checks and application-level retry logic. The agent application implements a ten-attempt retry loop with five-second intervals on initialization, including a `SELECT 1` connection verification before the schema is read. This ensures consistent behavior across restarts and eliminates the class of intermittent failures caused by the application reading an empty or partially initialized schema during database startup.
+
+### MCP Tool Server Architecture
+
+The Model Context Protocol server runs as a dedicated microservice on port 8701. Three categories of tools are registered at startup:
+
+**document_scanner.process** handles Tesseract OCR and is called at image upload time, not at query time. It returns a doc_id, the extracted text, and triggers persistence of both the OCR output and the image path to the document store.
+
+**granite_vision.qa** is the direct visual question-answering tool backed by Qwen 3.5 2B Vision via Ollama. It is called when a user submits a question with a doc_id and the original image file is available on disk. The internal tool identifier `granite_vision.qa` was retained from the original design specification; the underlying model was later replaced with Qwen 3.5 2B Vision for improved OCR accuracy and reduced VRAM usage, with no changes required to the agent orchestration code.
+
+**fast_sql tools** are fourteen schema-grounded SQL generation tools used exclusively at Level 3 of the QueryCraftAgent. Each tool queries the live database schema via SQLAlchemy's inspector to retrieve actual column names, then uses a deterministic `_build_sql()` function to construct a valid query, optionally verified by a short LLM call. The result is returned as a schema-validated SQL hint to the QueryCraftAgent, which passes it to the LLM for final refinement. This design means Level 3 failures are caused by query complexity or intent ambiguity, never by hallucinated schema details.
+
+---
+
+## Unified Multimodal Integration via MCP
+
+A defining architectural feature of this system is that OCR and multimodal reasoning are integrated through the same Model Context Protocol used for database access. The Qwen 3.5 2B Vision model is registered as a named MCP tool: `granite_vision.qa`. This internal tool identifier was retained from the original design specification; the underlying engine was subsequently replaced with Qwen 3.5 2B Vision, which demonstrated superior OCR accuracy and a smaller VRAM footprint. Because the tool interface is decoupled from its implementation, this substitution required no changes to the agent orchestration code — only the MCP tool registration was updated.
+
+This design delivers three concrete engineering properties:
+
+**Uniform Protocol:** The orchestrator issues identical-format tool calls regardless of whether the data source is a SQL table or a visual document. There are no special-case code paths for different modalities.
+
+**Modular Replaceability:** The vision model is encapsulated behind its MCP interface. Upgrading to a different vision model — such as a GPT-4V endpoint — requires updating only the tool registration, not any agent logic.
+
+**Cross-Modal Reasoning Capability:** Because both data sources share the same protocol, the agent can compose multi-step reasoning across modalities. For example, the system can extract a value from a document via the vision tool and immediately verify it against a database record via the SQL tool, within a single agentic workflow.
+
+```text
+Multimodal Request Flow
+
+Worker receives query with image path
+        |
+        +-- Image detected --> ocr_agent_qa()
+                                    |
+                                    +-- Unload Qwen 2.5 Coder (free VRAM)
+                                    |
+                                    +-- MCP tool call: granite_vision.qa
+                                                |
+                                            MCP Server (port 8701)
+                                                |
+                                            Qwen 3.5 2B Vision (Ollama)
+                                                |
+                                            Structured answer returned
+        |
+        +-- No image --> Standard Text-to-SQL pipeline (unchanged)
+```
+
+---
+
+## SQL Validation Pipeline
+
+The semantic validation layer is one of the most technically substantive components of the system. Rather than trusting LLM output directly, every generated SQL query passes through a deterministic validation pipeline before execution. Key checks include:
+
+- **Alias Integrity:** All table aliases used in SELECT, WHERE, JOIN, and GROUP BY clauses must be defined in FROM or JOIN declarations. The validator maintains a set of defined and used aliases and rejects queries with undefined references.
+- **Salary Table Recognition:** The `salary` table name was explicitly excluded from the SQL keyword blocklist, as it is both a legitimate table name and a column name in the employee schema. Incorrect inclusion in the keyword set caused the validator to silently discard valid salary table joins.
+- **Department Filter Enforcement:** Queries that reference a specific department by name are validated to confirm that a corresponding WHERE filter exists in the generated SQL. This prevents queries that return all departments when the user asked for one.
+- **PostgreSQL GROUP BY Compliance:** Queries containing aggregate functions are checked to ensure all non-aggregated SELECT columns appear in the GROUP BY clause, enforcing strict PostgreSQL semantics.
+- **Manager Schema Awareness:** Queries involving manager data are validated to confirm use of the `dept_manager` table rather than `dept_emp`. The validation produces a warning rather than a hard rejection, allowing the downstream semantic adjuster to regenerate the correct SQL before the query reaches the database.
+
+---
+
+## Research Methodology: Fine-Tuning and Model Selection
+
+As part of the project's scientific evaluation, a custom model fine-tuning pipeline was developed and benchmarked against the selected production model.
+
+**LoRA Fine-Tuning:** The base Qwen 2.5 3B model was fine-tuned using Low-Rank Adaptation on a specialized Text-to-SQL dataset.
+
+**Weight Merging:** The LoRA adapters were merged back into the base model weights via `merge_qwen_to_ollama.py` and quantized for local deployment via Ollama.
+
+**Comparative Evaluation:** The fine-tuned general model was benchmarked against the zero-shot `qwen2.5-coder:1.5b`, which is pre-trained on a large corpus of code and SQL.
+
+**Engineering Decision:** The benchmark results showed that the Coder model's zero-shot SQL accuracy exceeded that of the general LoRA fine-tune, with faster inference and lower error rates on complex multi-table joins. The production architecture was updated accordingly. This outcome demonstrates a data-driven engineering decision: fine-tuning is not always the optimal path when a domain-specialized pretrained model exists for the target task.
+
+See `Fine_Tuning_Qwen_2_5_3B.ipynb` and the `merged_qwen2_5_3b_text2sql` directory for the complete fine-tuning methodology and benchmark results.
+
+---
+
+## Technology Stack
+
+| Component | Technology |
+|---|---|
+| Core Framework | Python 3.11+, FastAPI, Uvicorn |
+| AI Orchestration | LlamaIndex, Model Context Protocol (MCP) |
+| Local Inference | Ollama (C++ optimized, memory-mapped) |
+| Text-to-SQL Model | qwen2.5-coder:1.5b |
+| Vision/OCR Model | qwen3.5:2b (registered as granite_vision.qa) |
+| Database ORM | SQLAlchemy |
+| Containerization | Docker, Docker Compose |
+| Databases | PostgreSQL, MySQL |
+
+---
+
+## Installation and Setup
+
+### Prerequisites
+
+- Ollama installed and running on the host machine
+- Python 3.11 or higher
+- Docker and Docker Compose
+- A supported database (PostgreSQL or MySQL)
+
+### 1. Pull Required Models
+
+```bash
+ollama pull qwen2.5-coder:1.5b
+ollama pull qwen3.5:2b
+```
+
+### 2. Configure Environment
+
+Install Python dependencies:
+
+```bash
+pip install -r requirements.txt
+```
+
+Create a `.env` file in the `Agent/` directory:
+
+```env
+DB_CONNECTION_URL=postgresql://user:password@localhost:5432/dbname
+OLLAMA_MODEL_NAME=qwen2.5-coder:1.5b
+GRANITE_MODEL_PATH=qwen3.5:2b
+```
+
+### 3. Run Without Docker
+
+Start each microservice in a separate terminal:
+
+```bash
+# Terminal 1 — Backend Worker
+python -u -m uvicorn "Agent.worker:app" --port 8700
+
+# Terminal 2 — MCP Tool Server
+python -m uvicorn "Agent.mcp_server:app" --port 8701
+
+# Terminal 3 — Web UI
+python -m uvicorn "web.app:app" --port 8000
+```
+
+Navigate to `http://localhost:8000` to interact with the system.
+
+### 4. Run With Docker
+
+Update `OLLAMA_BASE_URL` in `docker-compose.yml` to point to your host machine's Ollama instance, then run:
+
+```bash
+docker-compose up --build agent-app
+```
+
+The database containers include health checks and will be fully initialized before the application starts.
+
+---
+
+## Deployment on HuggingFace Spaces
+
+This section documents the **complete deployment journey** — from the first proof-of-concept tunnel approach to the final production-ready hybrid architecture. Each phase is explained with commands and rationale so this can serve as a reference for deploying any multi-modal AI application on HuggingFace Spaces.
+
+---
+
+### Understanding HuggingFace Spaces Constraints
+
+Before choosing a deployment strategy, it is essential to understand what the free tier of HuggingFace Spaces provides and restricts:
+
+| Resource | Free Tier |
+|----------|-----------|
+| CPU | 2 vCPUs |
+| RAM | 16 GB |
+| GPU | None |
+| Storage | Ephemeral (wiped on restart) |
+| Public port | 7860 only |
+| Timeout | Space sleeps after inactivity |
+
+The key challenge: **running a local LLM (e.g. `qwen2.5-coder:1.5b`) via Ollama consumes ~1.5–3 GB RAM, leaving limited headroom for vision models and the application stack.** Running a vision-language model such as `qwen3.5:2b` (~3 GB) alongside the text model causes out-of-memory (OOM) failures on the free CPU tier.
+
+---
+
+### Phase 1 — Cloudflare Tunnel: Bridging Local Resources to the Cloud
+
+#### Theory
+
+The simplest approach to deploy a resource-intensive app on HF Spaces is to **not run the heavy models in the cloud at all**. Instead, you run the LLM and VLM locally on your development machine and expose the Ollama API endpoint to the internet via a secure tunnel. The HF Space container merely hosts the web frontend and proxies inference requests back to your local machine.
+
+This approach has zero cloud compute cost and allows full GPU acceleration locally. However, it requires your development machine to remain **online and running at all times**, making it unsuitable for a persistent public demo.
+
+#### Setup
+
+**Step 1: Install Cloudflared on your local machine**
+```bash
+# Windows (via winget)
+winget install Cloudflare.cloudflared
+
+# Or download directly
+# https://github.com/cloudflare/cloudflared/releases
+```
+
+**Step 2: Start Ollama locally**
+```bash
+ollama serve
+# Ollama now listens on http://localhost:11434
+```
+
+**Step 3: Create a public tunnel to your local Ollama port**
+```bash
+cloudflared tunnel --url http://localhost:11434
+```
+
+Cloudflared will print a public HTTPS URL like:
+```
+https://your-random-id.trycloudflare.com
+```
+
+**Step 4: Set the tunnel URL as a secret in your HF Space**
+
+In the HF Space settings under **Variables and Secrets**, add:
+```
+OLLAMA_BASE_URL = https://your-random-id.trycloudflare.com
+```
+
+The application code reads `OLLAMA_BASE_URL` and routes all inference calls through the tunnel to your local GPU.
+
+#### Limitation
+
+- The tunnel URL changes every time `cloudflared` restarts, requiring a secret update.
+- Requires a continuously running local machine.
+- Not viable for a shared capstone demo or public deployment.
+
+---
+
+### Phase 2 — Self-Contained Docker: Running Ollama Inside the Container
+
+#### Theory
+
+The next evolution is to **embed Ollama directly into the Docker image** and pull models at container startup. This makes the deployment fully self-contained — no external machine is needed. The HF Space container becomes a complete inference server.
+
+The challenge is that HF Spaces rebuilds the Docker image on every push but does **not cache downloaded model weights** between restarts (storage is ephemeral). This means every **Factory Reboot** triggers a fresh ~1–3 GB model download, causing a cold-start delay of 2–5 minutes.
+
+To prevent HF's health monitor from timing out the Space during this download, model pulling is performed **asynchronously in the background**, allowing the web UI to become responsive immediately while the model fetches in parallel.
+
+#### Dockerfile Changes
+
+Add Ollama installation to the `Dockerfile`:
+
+```dockerfile
+# Install Ollama binary (pinned to a stable release)
+RUN curl -L https://github.com/ollama/ollama/releases/download/v0.5.11/ollama-linux-amd64.tgz \
+    -o ollama.tgz && \
+    tar -C /usr -xzf ollama.tgz && \
+    rm ollama.tgz
+```
+
+#### Startup Script (`start_hf.sh`)
+
+```bash
+#!/bin/bash
+export HOME=/root
+
+echo "[HF-INIT] Starting Ollama server..."
+ollama serve &
+OLLAMA_PID=$!
+
+# Wait for Ollama to become healthy (up to 20 attempts)
+for i in $(seq 1 20); do
+    if curl -sf http://localhost:11434/api/tags > /dev/null 2>&1; then
+        echo "[HF-INIT] Ollama is ALIVE!"
+        break
+    fi
+    echo "[HF-INIT] Waiting... ($i/20)"
+    sleep 3
+done
+
+# Pull the text model asynchronously (non-blocking)
+echo "[HF-INIT] Pulling qwen2.5-coder:1.5b in background..."
+ollama pull qwen2.5-coder:1.5b &
+
+# Start application services immediately (do not wait for model pull)
+echo "[HF-INIT] Starting Backend Worker..."
+python -m uvicorn Agent.worker:app --host 0.0.0.0 --port 8700 &
+
+echo "[HF-INIT] Starting MCP Tool Server..."
+python -m uvicorn Agent.mcp_server:app --host 0.0.0.0 --port 8701 &
+
+echo "[HF-INIT] Starting Web UI on port 7860..."
+exec python -m uvicorn web.app:app --host 0.0.0.0 --port 7860
+```
+
+The `&` (ampersand) operator sends the model pull to the background. The `exec` on the last line replaces the shell process with the web server, ensuring HF's process monitor tracks the correct PID.
+
+#### Limitation
+
+- **Vision model OOM**: Adding `qwen3.5:2b` alongside `qwen2.5-coder:1.5b` exceeds the 16 GB RAM limit when both are loaded simultaneously.
+- Model weights are re-downloaded on every Factory Reboot.
+- Ollama version pinning is critical — newer models require newer Ollama binaries and pulling the wrong combination causes `412` errors.
+
+---
+
+### Phase 3 — Hybrid Serverless API (Final Production Architecture)
+
+#### Theory
+
+The final and most robust architecture eliminates the in-container LLM entirely for text generation and uses **external serverless inference APIs** instead. The container only runs lightweight services (web UI, API proxy, Tesseract OCR), while all neural inference is delegated to dedicated providers:
+
+| Task | Provider | Model | Cost |
+|------|----------|-------|------|
+| Text-to-SQL | HuggingFace Inference API | `Qwen/Qwen2.5-Coder-7B-Instruct` | Free tier |
+| Vision Q&A | Groq API | `meta-llama/llama-4-scout-17b-16e-instruct` | Free (14,400 req/day) |
+| OCR | Tesseract (in container) | — | Free |
+| Database | Neon PostgreSQL | — | Free tier |
+
+This eliminates OOM risk entirely, removes the model download cold-start, and makes the container startup under 30 seconds.
+
+#### Why Groq for Vision Instead of HuggingFace Serverless?
+
+HuggingFace's serverless inference for vision-language models has two limitations on the free tier:
+1. Most VLM models are not routed through any enabled provider for free-tier accounts.
+2. Google Gemini's free tier blocks requests originating from cloud datacenter IP ranges (such as HF's infrastructure), setting the effective quota to zero even for valid free-tier API keys.
+
+**Groq** is purpose-built for API-first server deployments and explicitly allows free-tier requests from cloud IPs. It provides 14,400 free requests per day with no IP restrictions.
+
+#### Required HuggingFace Space Secrets
+
+Set these in **Space Settings → Variables and Secrets**:
+
+```
+USE_HF_CLOUD       = 1
+HF_API_TOKEN       = # HuggingFace access token
+GROQ_API_KEY       = # From console.groq.com (free)
+DB_CONNECTION_URL  = # Neon PostgreSQL connection string
+```
+
+#### Obtaining API Keys
+
+**HuggingFace Token:**
+1. Go to https://huggingface.co/settings/tokens
+2. Create a token with `read` scope (or `write` if you need to push models)
+
+**Groq API Key:**
+1. Go to https://console.groq.com
+2. Sign in with Google / GitHub
+3. Navigate to **API Keys** → **Create API Key**
+4. Copy the key
+
+**Neon PostgreSQL:**
+1. Go to https://neon.tech
+2. Create a free project
+3. Copy the connection string from the dashboard
+
+#### Slim Dockerfile (No Ollama Required)
+
+```dockerfile
+FROM python:3.11-slim
+
+ENV PYTHONDONTWRITEBYTECODE=1
+ENV PYTHONUNBUFFERED=1
+ENV USE_HF_CLOUD=1
+
+RUN apt-get update && apt-get install -y \
+    curl \
+    tesseract-ocr \
+    libtesseract-dev \
+    gcc \
+    libpq-dev \
+    && rm -rf /var/lib/apt/lists/*
+
+WORKDIR /app
+COPY requirements.txt .
+RUN pip install --no-cache-dir -r requirements.txt
+
+COPY . .
+RUN useradd -m -u 1000 appuser && chown -R appuser:appuser /app
+USER appuser
+
+CMD ["bash", "start_hf.sh"]
+```
+
+#### Key `requirements.txt` Additions
+
+```txt
+huggingface_hub      # HF Inference API client
+groq>=0.9.0          # Groq Vision API client
+google-genai>=1.0.0  # Optional: Google Gemini (requires billing for cloud IPs)
+pytesseract>=0.3.10  # Local OCR fallback
+pillow>=10.0.0
+```
+
+#### Deployment Steps
+
+1. **Create a HuggingFace Space** at https://huggingface.co/spaces
+   - SDK: **Docker**
+   - Visibility: Public or Private
+
+2. **Push your repository** to the Space's git remote:
+   ```bash
+   git remote add space https://huggingface.co/spaces/<your-username>/<space-name>
+   git push space main
+   ```
+
+3. **Add secrets** in Space Settings (see table above). Do **not** hardcode secrets in the Dockerfile or any committed file.
+
+4. **Factory Reboot** the Space after adding secrets to trigger a clean rebuild with the new environment variables.
+
+5. **Monitor the build logs** — a successful startup shows:
+   ```
+   [GRANITE_VQA] Target Model: meta-llama/llama-4-scout-17b-16e-instruct (via Groq Vision)
+   [INFO] CLOUD MODE: Initializing Official HF InferenceClient (Qwen/Qwen2.5-Coder-7B-Instruct)
+   INFO: Uvicorn running on http://0.0.0.0:7860
+   ```
+
+#### Important Notes on Line Endings
+
+HuggingFace containers run Linux. If you develop on Windows, your shell scripts will have `\r\n` (CRLF) line endings, which cause `bad interpreter` errors on Linux. Add this to your `Dockerfile` to strip them automatically:
+
+```dockerfile
+RUN sed -i 's/\r$//' start_hf.sh && chmod +x start_hf.sh
+```
+
+Alternatively, configure Git to normalize line endings:
+```bash
+git config core.autocrlf input
+```
+
+
+---
+
+## Project Summary
+
+This project demonstrates that a self-correcting, multi-modal AI pipeline can be built and deployed entirely on consumer hardware through careful architectural decisions: progressive SQL generation with semantic validation, unified MCP-based tool access across data modalities, dynamic VRAM orchestration to prevent memory exhaustion, and a hybrid containerization strategy that preserves native GPU performance while maintaining deployment portability.
+
+The system was developed iteratively, with each architectural decision — from model selection to validation logic to infrastructure design — informed by observed failure modes and measured outcomes rather than theoretical assumptions.
+
+---
+*Built with ❤️ for advanced Agentic Data Engineering.*

docker-compose.yml

@@ -0,0 +1,79 @@
+version: '3.8'
+
+services:
+  # Main Agentic Application
+  agent-app:
+    build: .
+    container_name: text2sql-agent-app
+    restart: unless-stopped
+    env_file: Agent/.env
+    environment:
+      - OLLAMA_BASE_URL=http://172.27.224.1:11434
+      - LLM_CTX=4096
+    deploy:
+      resources:
+        limits:
+          cpus: '2.0'
+          memory: 4G
+    ports:
+      - "8000:8000"
+      - "8700:8700"
+      - "8701:8701"
+    depends_on:
+      employee-postgres:
+        condition: service_healthy
+      employee-mysql:
+        condition: service_healthy
+    volumes:
+      - ./Agent:/app/Agent
+      - ./web:/app/web
+    networks:
+      - employee-net
+
+  # MySQL Database
+  employee-mysql:
+    image: mysql:8.0
+    container_name: employee-mysql
+    restart: unless-stopped
+    env_file: Agent/.env
+    ports:
+      - "${MYSQL_PORT:-3306}:3306"
+    volumes:
+      - ./employee-sample-database/mysql/dataset_small:/docker-entrypoint-initdb.d
+      - mysql-data:/var/lib/mysql
+    networks:
+      - employee-net
+    healthcheck:
+      test: ["CMD", "mysqladmin", "ping", "-h", "localhost", "--silent"]
+      interval: 5s
+      timeout: 5s
+      retries: 12
+      start_period: 30s
+
+  # PostgreSQL Database
+  employee-postgres:
+    image: postgres:15
+    container_name: employee-postgres
+    restart: unless-stopped
+    env_file: Agent/.env
+    ports:
+      - "${POSTGRES_PORT:-5432}:5432"
+    volumes:
+      - ./employee-sample-database/postgres/dataset_small:/docker-entrypoint-initdb.d
+      - postgres-data:/var/lib/postgresql/data
+    networks:
+      - employee-net
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -U $${POSTGRES_USER:-postgres}"]
+      interval: 5s
+      timeout: 5s
+      retries: 12
+      start_period: 30s
+
+volumes:
+  mysql-data:
+  postgres-data:
+
+networks:
+  employee-net:
+    driver: bridge

requirements.txt

@@ -0,0 +1,38 @@
+# Core Frameworks
+fastapi>=0.100.0
+uvicorn>=0.23.0
+sqlalchemy>=2.0.0
+pydantic>=2.0.0
+python-multipart>=0.0.6
+pandas>=2.0.0
+
+# Multi-Agent Orchestration
+llama-index>=0.10.0
+llama-index-core>=0.10.0
+llama-index-llms-ollama>=0.1.2
+llama-index-llms-huggingface-api
+huggingface_hub
+
+# Model Context Protocol
+mcp>=0.1.0
+sqlparse>=0.4.4
+
+# HTTP & Async
+requests>=2.31.0
+aiohttp>=3.9.0
+psutil>=5.9.0
+
+# Database Drivers
+pymysql>=1.1.0
+asyncpg>=0.29.0
+psycopg2-binary>=2.9.0
+aiomysql>=0.2.0
+
+# Document Vision processing
+pillow>=10.0.0
+pytesseract>=0.3.10
+google-genai>=1.0.0
+groq>=0.9.0
+
+# Environment
+python-dotenv>=1.0.0

start_hf.sh

@@ -0,0 +1,57 @@
+#!/bin/bash
+
+# Cloud-Optimized Startup Script for Hugging Face Spaces
+# Model is pre-baked into the image — no runtime pull needed.
+
+# 0. Start Ollama server
+echo "[HF-INIT] Starting Ollama server..."
+export HOME=/app
+export OLLAMA_HOST=0.0.0.0:11434
+export OLLAMA_MODELS=/app/ollama_models
+
+ollama serve > /app/ollama.log 2>&1 &
+
+# Wait for Ollama to be ready
+echo "[HF-INIT] Waiting for Ollama to start..."
+for i in {1..20}; do
+    if curl -s http://localhost:11434/api/tags > /dev/null 2>&1; then
+        echo "[HF-INIT] Ollama is ready."
+        break
+    fi
+    if [ $i -eq 10 ]; then
+        echo "[HF-INIT] Still waiting — Ollama log:"
+        cat /app/ollama.log
+    fi
+    echo "[HF-INIT] Waiting... ($i/20)"
+    sleep 2
+done
+
+# Verify the pre-baked model is present before starting the app.
+# If it is missing for any reason, pull it now (blocking, so app waits).
+echo "[HF-INIT] Verifying qwen3.5:2b is available..."
+if ollama list 2>/dev/null | grep -q "qwen3.5:2b"; then
+    echo "[HF-INIT] qwen3.5:2b confirmed present."
+else
+    echo "[HF-INIT] WARNING: qwen3.5:2b not found in pre-baked layers. Pulling now..."
+    ollama pull qwen3.5:2b
+    if [ $? -eq 0 ]; then
+        echo "[HF-INIT] Pull succeeded."
+    else
+        echo "[HF-INIT] ERROR: Pull failed. Vision Q&A will not work."
+    fi
+fi
+
+# 1. Start Backend Worker
+echo "[HF-INIT] Starting Backend Worker on port 8700..."
+python -u -m uvicorn "Agent.worker:app" --host 0.0.0.0 --port 8700 &
+
+# 2. Start MCP Tool Server
+echo "[HF-INIT] Starting MCP Tool Server on port 8701..."
+python -m uvicorn "Agent.mcp_server:app" --host 0.0.0.0 --port 8701 &
+
+# 3. Start Web UI
+echo "[HF-INIT] Starting Web UI on port 7860..."
+python -m uvicorn "web.app:app" --host 0.0.0.0 --port 7860 &
+
+# Keep the container alive
+wait

web/app.py

@@ -0,0 +1,170 @@
+import os
+import httpx
+import logging
+import traceback
+from typing import Optional
+from fastapi import FastAPI, HTTPException, Request
+from pydantic import BaseModel
+from fastapi.staticfiles import StaticFiles
+from fastapi.responses import FileResponse, JSONResponse
+from fastapi.middleware.cors import CORSMiddleware
+
+WORKER_URL = os.environ.get("WORKER_URL", "http://localhost:8700")
+# Timeout (seconds) when calling the worker; re-execute and heavy queries can take longer.
+# Make configurable via environment variable WORKER_TIMEOUT (default 300s).
+WORKER_TIMEOUT = float(os.environ.get("WORKER_TIMEOUT", "450"))  # Increased to 450s to handle complex queries
+
+# Reusable AsyncClient to avoid per-request connection setup overhead
+client: httpx.AsyncClient | None = None
+
+app = FastAPI(title="Text2SQL Web API")
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=["*"],  # adjust in production
+    allow_credentials=True,
+    allow_methods=["*"],
+    allow_headers=["*"],
+)
+
+# Serve static frontend under /static and provide a root route
+static_dir = os.path.join(os.path.dirname(__file__), 'static')
+if os.path.isdir(static_dir):
+    app.mount("/static", StaticFiles(directory=static_dir, html=True), name="static")
+
+
+@app.get("/")
+async def root_index():
+    index_path = os.path.join(static_dir, 'index.html')
+    return FileResponse(index_path)
+
+class QueryReq(BaseModel):
+    question: str
+    # Optional doc_id produced by a prior /upload-image call
+    doc_id: Optional[str] = None
+    # Explicit mode hint from client: 'text2sql' or 'ocr_qa'. When omitted, worker may auto-detect.
+    mode: Optional[str] = None
+
+class OcrQAReq(BaseModel):
+    question: str
+    doc_id: str
+
+class OcrQAResp(BaseModel):
+    doc_id: str
+    answer: str
+    used_llm: bool
+
+@app.get('/health')
+async def health():
+    return {"status": "ok"}
+
+@app.post('/api/query')
+async def api_query(req: QueryReq):
+    # Use a shared AsyncClient created at startup to avoid TCP/TLS/HTTP handshakes per request.
+    global client
+    if client is None:
+        # fallback to create a temporary client (shouldn't happen if startup ran)
+        client = httpx.AsyncClient(timeout=WORKER_TIMEOUT)
+    try:
+        payload = {"question": req.question}
+        # Forward explicit mode if provided so the worker doesn't infer from lingering doc_id
+        if getattr(req, 'mode', None):
+            payload['mode'] = req.mode
+        if getattr(req, 'doc_id', None):
+            payload['doc_id'] = req.doc_id
+        r = await client.post(f"{WORKER_URL}/query", json=payload)
+        r.raise_for_status()
+        return r.json()
+    except httpx.HTTPStatusError as e:
+        # Log the full traceback and response body for debugging
+        logging.error("Worker returned non-2xx status: %s", e.response.status_code)
+        logging.error("Response text: %s", getattr(e.response, 'text', str(e)))
+        logging.error(traceback.format_exc())
+        raise HTTPException(status_code=e.response.status_code, detail=getattr(e.response, 'text', str(e)))
+    except httpx.ReadTimeout as e:
+        # Worker took too long. Return a 504 gateway timeout with a helpful message.
+        logging.error("Timeout contacting worker after %ss: %s", WORKER_TIMEOUT, str(e))
+        logging.error(traceback.format_exc())
+        return JSONResponse(status_code=504, content={
+            "error": "worker_timeout",
+            "message": f"Worker did not respond within {WORKER_TIMEOUT} seconds. The query may be long-running (re-execute) or the worker is busy. Try increasing WORKER_TIMEOUT or running the worker with more resources.",
+        })
+    except Exception as e:
+        logging.error("Error contacting worker: %s", str(e))
+        logging.error(traceback.format_exc())
+        # Return a JSON response containing the error detail to help debugging from the client
+        return JSONResponse(status_code=500, content={"error": str(e), "trace": traceback.format_exc()})
+
+
+@app.post('/api/ocr-qa')
+async def api_ocr_qa(req: OcrQAReq):
+    global client
+    if client is None:
+        client = httpx.AsyncClient(timeout=WORKER_TIMEOUT)
+    try:
+        payload = {"question": req.question, "doc_id": req.doc_id}
+        r = await client.post(f"{WORKER_URL}/ocr-qa", json=payload)
+        r.raise_for_status()
+        return r.json()
+    except httpx.HTTPStatusError as e:
+        logging.error("Worker OCR QA non-2xx: %s", e.response.status_code)
+        logging.error("Response text: %s", getattr(e.response, 'text', str(e)))
+        return JSONResponse(status_code=e.response.status_code, content={"error": getattr(e.response, 'text', str(e))})
+    except Exception as e:
+        logging.error("Error contacting worker OCR QA: %s", str(e))
+        return JSONResponse(status_code=500, content={"error": str(e), "trace": traceback.format_exc()})
+
+
+@app.on_event('startup')
+async def startup_event():
+    global client
+    if client is None:
+        client = httpx.AsyncClient(timeout=WORKER_TIMEOUT)
+
+
+@app.on_event('shutdown')
+async def shutdown_event():
+    global client
+    if client is not None:
+        await client.aclose()
+
+
+@app.post('/api/test')
+async def api_test(req: QueryReq):
+    # Simple echo endpoint to verify POST handling without contacting the worker
+    return {"ok": True, "echo": req.question}
+
+
+@app.post('/upload-image')
+async def upload_image(file: bytes = None, request: Request = None):
+    """Proxy an image upload to the worker's /upload-image endpoint.
+
+    Accepts multipart/form-data file field 'file' from the browser.
+    Returns the worker JSON (doc_id, text_preview, image_url) on success.
+    """
+    # Use the request.form() to access the uploaded file
+    try:
+        form = await request.form()
+        f = form.get('file')
+        if f is None:
+            raise HTTPException(status_code=400, detail='file field is required')
+        # forward the file to the worker
+        files = {'file': (f.filename, await f.read(), f.content_type)}
+        async with httpx.AsyncClient(timeout=120) as c:
+            r = await c.post(f"{WORKER_URL}/upload-image", files=files)
+            r.raise_for_status()
+            return JSONResponse(status_code=200, content=r.json())
+    except HTTPException:
+        raise
+    except Exception as e:
+        logging.error('upload-image proxy failed: %s', str(e))
+        raise HTTPException(status_code=500, detail=str(e))
+
+
+@app.get('/api/test')
+async def api_test_get():
+    return {"ok": True, "msg": "POST to /api/test with JSON {question: ...} to echo"}
+
+if __name__ == '__main__':
+    import uvicorn
+    port = int(os.environ.get('WEB_PORT', 8000))
+    uvicorn.run(app, host='0.0.0.0', port=port)

web/static/app.js

@@ -0,0 +1,584 @@
+const askBtn = document.getElementById('ask');
+const questionEl = document.getElementById('question');
+const answerEl = document.getElementById('answer');
+const sqlEl = document.getElementById('sql');
+const memoryEl = document.getElementById('memory');
+const querycraftEl = document.getElementById('querycraft');
+const presenterEl = document.getElementById('presenter');
+const uploadInput = document.getElementById('upload');
+const uploadBtn = document.getElementById('uploadBtn');
+const uploadStatus = document.getElementById('uploadStatus');
+const uploadControls = document.getElementById('uploadControls');
+const modeToggle = document.getElementById('modeToggle');
+let currentDocId = null;
+let currentMode = 'text2sql';
+const statusEl = document.getElementById('status');
+const timeEl = document.getElementById('time');
+const recalcBtn = document.getElementById('recalc');
+const spinner = document.getElementById('spinner');
+const spinnerText = document.getElementById('spinnerText');
+const askBtnEl = askBtn;
+
+// Mode toggle logic
+modeToggle.addEventListener('change', function() {
+  currentMode = modeToggle.value;
+  console.log('[DEBUG] Mode changed to:', currentMode); // Debug log
+  if(currentMode === 'ocr_qa') {
+    uploadControls.classList.remove('hidden');
+    console.log('[DEBUG] Upload controls shown'); // Debug log
+  } else {
+    uploadControls.classList.add('hidden');
+    currentDocId = null;
+    uploadStatus.textContent = 'No upload';
+    uploadStatus.className = 'status-text';
+    console.log('[DEBUG] Upload controls hidden'); // Debug log
+  }
+});
+
+// Initialize UI state
+console.log('[DEBUG] Initial mode:', currentMode); // Debug log
+if(currentMode === 'ocr_qa') {
+  uploadControls.classList.remove('hidden');
+} else {
+  uploadControls.classList.add('hidden');
+}
+
+// Helper to programmatically switch to OCR Q&A mode
+function switchToOcrQaMode() {
+  if (modeToggle.value !== 'ocr_qa') {
+    modeToggle.value = 'ocr_qa';
+    currentMode = 'ocr_qa';
+    uploadControls.classList.remove('hidden');
+    // Fire a change event so any listeners update state consistently
+    const ev = new Event('change', { bubbles: true });
+    modeToggle.dispatchEvent(ev);
+    console.log('[DEBUG] Switched to OCR Q&A mode');
+  }
+}
+
+// Extract mode change logic into a function so it can be called directly
+function onModeChange() {
+  currentMode = modeToggle.value;
+  console.log('[DEBUG] onModeChange called, mode:', currentMode); // Debug log
+  if(currentMode === 'ocr_qa') {
+    uploadControls.classList.remove('hidden');
+    console.log('[DEBUG] Upload controls shown via onModeChange'); // Debug log
+  } else {
+    uploadControls.classList.add('hidden');
+    currentDocId = null;
+    uploadStatus.textContent = 'No upload';
+    uploadStatus.className = 'status-text';
+    console.log('[DEBUG] Upload controls hidden via onModeChange'); // Debug log
+  }
+}
+
+// Mode toggle logic
+modeToggle.addEventListener('change', onModeChange);
+
+// Upload handler: create a temporary file input on demand to avoid issues with a permanently hidden input.
+if (uploadBtn) {
+  uploadBtn.addEventListener('click', () => {
+    // If not in OCR Q&A mode, switch and highlight, then open file picker after a delay
+    if (modeToggle.value !== 'ocr_qa') {
+      switchToOcrQaMode();
+      // Visually highlight the dropdown to show the switch
+      modeToggle.style.boxShadow = '0 0 0 3px rgba(15, 98, 254, 0.2)';
+      modeToggle.style.transition = 'box-shadow 0.2s';
+      setTimeout(() => {
+        modeToggle.style.boxShadow = '';
+        // Now open the file picker
+        const tmp = document.createElement('input');
+        tmp.type = 'file';
+        tmp.accept = 'image/*';
+        tmp.style.position = 'absolute';
+        tmp.style.left = '-9999px';
+        document.body.appendChild(tmp);
+
+        tmp.addEventListener('change', async () => {
+          const files = tmp.files;
+          if (!files || files.length === 0) {
+            uploadStatus.textContent = 'No file selected';
+            document.body.removeChild(tmp);
+            return;
+          }
+          switchToOcrQaMode();
+          uploadStatus.textContent = 'Uploading...';
+          uploadStatus.className = 'status-text text-warning';
+          const fd = new FormData();
+          fd.append('file', files[0]);
+          try {
+            const r = await fetch('/upload-image', { method: 'POST', body: fd });
+            if (!r.ok) {
+              const j = await r.json().catch(()=>({}));
+              uploadStatus.textContent = 'Upload failed: ' + (j.detail || r.statusText);
+              uploadStatus.className = 'status-text text-error';
+              document.body.removeChild(tmp);
+              return;
+            }
+            const j = await r.json();
+            currentDocId = j.doc_id;
+            uploadStatus.textContent = `Uploaded (doc_id=${currentDocId})`;
+            uploadStatus.className = 'status-text text-success';
+          } catch (e) {
+            uploadStatus.textContent = 'Upload error: ' + String(e);
+            uploadStatus.className = 'status-text text-error';
+          }
+          document.body.removeChild(tmp);
+        });
+        tmp.click();
+      }, 350); // 350ms to allow UI to update and user to see the switch
+      return;
+    }
+    // Already in OCR Q&A mode: open file picker immediately
+    const tmp = document.createElement('input');
+    tmp.type = 'file';
+    tmp.accept = 'image/*';
+    tmp.style.position = 'absolute';
+    tmp.style.left = '-9999px';
+    document.body.appendChild(tmp);
+
+    tmp.addEventListener('change', async () => {
+      const files = tmp.files;
+      if (!files || files.length === 0) {
+        uploadStatus.textContent = 'No file selected';
+        uploadStatus.className = 'status-text';
+        document.body.removeChild(tmp);
+        return;
+      }
+      switchToOcrQaMode();
+      uploadStatus.textContent = 'Uploading...';
+      uploadStatus.className = 'status-text text-warning';
+      const fd = new FormData();
+      fd.append('file', files[0]);
+      try {
+        const r = await fetch('/upload-image', { method: 'POST', body: fd });
+        if (!r.ok) {
+          const j = await r.json().catch(()=>({}));
+          uploadStatus.textContent = 'Upload failed: ' + (j.detail || r.statusText);
+          uploadStatus.className = 'status-text text-error';
+          document.body.removeChild(tmp);
+          return;
+        }
+        const j = await r.json();
+        currentDocId = j.doc_id;
+        uploadStatus.textContent = `Uploaded (doc_id=${currentDocId})`;
+        uploadStatus.className = 'status-text text-success';
+      } catch (e) {
+        uploadStatus.textContent = 'Upload error: ' + String(e);
+        uploadStatus.className = 'status-text text-error';
+      }
+      document.body.removeChild(tmp);
+    });
+    tmp.click();
+  });
+}
+
+function setRunning(running){
+  if(running){
+    askBtnEl.disabled = true;
+    recalcBtn.disabled = true;
+    questionEl.disabled = true;
+    spinner.classList.remove('hidden');
+    spinnerText.classList.remove('hidden');
+    spinnerText.textContent = 'Processing...';
+    // hide the small status text while spinner is active
+    statusEl.textContent = '';
+    console.log('[DEBUG] Loading state: ON, spinner should be visible'); // Debug log
+  } else {
+    askBtnEl.disabled = false;
+    recalcBtn.disabled = false;
+    questionEl.disabled = false;
+    spinner.classList.add('hidden');
+    spinnerText.classList.add('hidden');
+    // restore default status
+    statusEl.textContent = 'Ready';
+    console.log('[DEBUG] Loading state: OFF, spinner should be hidden'); // Debug log
+  }
+}
+
+function renderPresenterTrace(text){
+  if(!text) return '—';
+  // Split traces into chunks separated by blank lines and render as collapsible
+  const parts = String(text).split(/\n\s*\n/).filter(Boolean);
+  const container = document.createElement('div');
+  parts.forEach((p, i) => {
+    const d = document.createElement('details');
+    const summary = document.createElement('summary');
+    summary.textContent = p.split('\n')[0].slice(0,80);
+    summary.style.cursor = 'pointer';
+    summary.style.padding = '0.5rem';
+    summary.style.borderRadius = '0.375rem';
+    summary.style.marginBottom = '0.5rem';
+    summary.style.backgroundColor = 'var(--bg-tertiary)';
+    const pre = document.createElement('pre');
+    pre.style.whiteSpace = 'pre-wrap';
+    pre.style.fontSize = '0.75rem';
+    pre.style.padding = '0.75rem';
+    pre.style.backgroundColor = 'var(--bg-secondary)';
+    pre.style.borderRadius = '0.375rem';
+    pre.style.margin = '0';
+    pre.textContent = p;
+    d.appendChild(summary);
+    d.appendChild(pre);
+    if(i===0) d.open = true;
+    container.appendChild(d);
+  });
+  return container;
+}
+
+// small helper to escape HTML
+const esc = s => String(s || '').replace(/&/g,'&amp;').replace(/</g,'&lt;').replace(/>/g,'&gt;');
+
+// Detect if text contains a pipe/markdown table and return true
+function isMarkdownTable(text){
+  if(!text) return false;
+  const lines = String(text).split(/\r?\n/).map(l=>l.trim()).filter(Boolean);
+  if(lines.length < 2) return false;
+  // Count lines that contain pipe separators
+  const pipeLines = lines.filter(l => l.includes('|'));
+  if(pipeLines.length < 2) return false;
+  // If the second line is a markdown separator like | --- | --- | or ---|---
+  const second = lines[1];
+  if(/^\s*\|?\s*:?-{3,}:?\s*(\|\s*:?-{3,}:?\s*)+\|?\s*$/.test(second)) return true;
+  // Otherwise, if majority of non-empty lines contain pipes, treat as table
+  return (pipeLines.length / lines.length) >= 0.5;
+}
+
+// Parse a simple markdown/pipe table into a DOM <table>
+function parseMarkdownTable(text){
+  const lines = String(text).split(/\r?\n/).map(l=>l.trim()).filter(Boolean);
+  // find the first contiguous block of pipe lines
+  let start = -1, end = -1;
+  for(let i=0;i<lines.length;i++){
+    if(lines[i].includes('|')){ start = i; break; }
+  }
+  if(start === -1) return null;
+  for(let i=start;i<lines.length;i++){
+    if(!lines[i].includes('|')){ end = i; break; }
+  }
+  if(end === -1) end = lines.length;
+  const block = lines.slice(start, end);
+  const rows = block.map(r => {
+    let s = r;
+    if(s.startsWith('|')) s = s.slice(1);
+    if(s.endsWith('|')) s = s.slice(0,-1);
+    return s.split('|').map(c => c.trim());
+  });
+
+  // If second row is separator, treat first row as header
+  let header = null;
+  if(rows.length >= 2 && rows[1].every(c => /^:?-{3,}:?$/.test(c))) {
+    header = rows[0];
+    rows.splice(0,2);
+  }
+
+  const table = document.createElement('table');
+  table.className = 'answer-table';
+  const thead = document.createElement('thead');
+  const tbody = document.createElement('tbody');
+
+  if(header){
+    const tr = document.createElement('tr');
+    header.forEach(h => { const th = document.createElement('th'); th.textContent = h || ''; tr.appendChild(th); });
+    thead.appendChild(tr);
+  } else if(rows.length>0){
+    // use first row as header if no separator present
+    const tr = document.createElement('tr');
+    rows[0].forEach((_,i) => { const th = document.createElement('th'); th.textContent = `col ${i+1}`; tr.appendChild(th); });
+    thead.appendChild(tr);
+  }
+
+  rows.forEach(r => {
+    const tr = document.createElement('tr');
+    r.forEach(c => { const td = document.createElement('td'); td.textContent = c; tr.appendChild(td); });
+    tbody.appendChild(tr);
+  });
+
+  if(thead.childElementCount) table.appendChild(thead);
+  table.appendChild(tbody);
+  
+  // Create enhanced table wrapper with controls
+  const wrapper = document.createElement('div');
+  wrapper.className = 'table-wrapper';
+  
+  // Create table controls
+  const controls = document.createElement('div');
+  controls.className = 'table-controls';
+  
+  const controlsLeft = document.createElement('div');
+  controlsLeft.className = 'table-controls-left';
+  
+  const tableInfo = document.createElement('div');
+  tableInfo.className = 'table-info';
+  const rowCount = tbody.children.length;
+  const colCount = header ? header.length : (rows.length > 0 ? rows[0].length : 0);
+  tableInfo.textContent = `${rowCount} rows × ${colCount} columns`;
+  
+  controlsLeft.appendChild(tableInfo);
+  
+  const controlsRight = document.createElement('div');
+  controlsRight.className = 'table-controls-right';
+  
+  const resizeControls = document.createElement('div');
+  resizeControls.className = 'table-resize-controls';
+  
+  // Size control buttons
+  const sizes = [
+    { name: 'Compact', class: 'compact' },
+    { name: 'Comfortable', class: 'comfortable' },
+    { name: 'Spacious', class: 'spacious' }
+  ];
+  
+  sizes.forEach((size, index) => {
+    const btn = document.createElement('button');
+    btn.textContent = size.name;
+    btn.className = 'table-resize-btn';
+    if (index === 1) btn.classList.add('active'); // Default to comfortable
+    
+    btn.addEventListener('click', () => {
+      // Remove all size classes
+      table.classList.remove('compact', 'comfortable', 'spacious');
+      // Add selected size class
+      table.classList.add(size.class);
+      
+      // Update active button
+      resizeControls.querySelectorAll('.table-resize-btn').forEach(b => b.classList.remove('active'));
+      btn.classList.add('active');
+    });
+    
+    resizeControls.appendChild(btn);
+  });
+  
+  controlsRight.appendChild(resizeControls);
+  
+  controls.appendChild(controlsLeft);
+  controls.appendChild(controlsRight);
+  
+  // Create table container
+  const container = document.createElement('div');
+  container.className = 'table-container';
+  container.appendChild(table);
+  
+  wrapper.appendChild(controls);
+  wrapper.appendChild(container);
+  
+  // Set default size
+  table.classList.add('comfortable');
+  
+  return wrapper;
+}
+
+// heuristics: split logs into agent sections
+function splitLogs(logs) {
+  const sections = {memory: '', querycraft: '', presenter: ''};
+  if (!logs) {
+    console.log('[DEBUG] splitLogs: No logs provided');
+    return sections;
+  }
+  
+  console.log('[DEBUG] splitLogs: Input logs length:', logs.length);
+  console.log('[DEBUG] splitLogs: First 500 chars:', logs.substring(0, 500));
+  console.log('[DEBUG] splitLogs: Last 500 chars:', logs.substring(Math.max(0, logs.length - 500)));
+  
+  // Normalize line endings
+  const lines = logs.replace(/\r/g,'').split('\n');
+  console.log('[DEBUG] splitLogs: Total lines:', lines.length);
+
+  // Look for explicit markers first
+  let mode = null;
+  let memoryMarkerFound = false;
+  let querycraftMarkerFound = false;
+  let presenterMarkerFound = false;
+  
+  for (let i = 0; i < lines.length; i++) {
+    const line = lines[i];
+    const l = line.trim();
+    
+    // Match the exact agent section headers from agentic_workflow.py
+    if (/=== MEMORY AGENT ===/i.test(l) || /INITIAL MEMORY/i.test(l)) { 
+      mode = 'memory'; 
+      memoryMarkerFound = true;
+      console.log(`[DEBUG] splitLogs: Found MEMORY marker at line ${i}: "${l}"`);
+      continue; 
+    }
+    if (/=== QUERY CRAFT AGENT ===/i.test(l) || /QUERY_CRAFT/i.test(l) || /MICRO-HINT/i.test(l)) { 
+      mode = 'querycraft'; 
+      querycraftMarkerFound = true;
+      console.log(`[DEBUG] splitLogs: Found QUERYCRAFT marker at line ${i}: "${l}"`);
+      continue; 
+    }
+    if (/=== RESULT PRESENTER AGENT ===/i.test(l) || /PRESENTER/i.test(l)) { 
+      mode = 'presenter'; 
+      presenterMarkerFound = true;
+      console.log(`[DEBUG] splitLogs: Found PRESENTER marker at line ${i}: "${l}"`);
+      continue; 
+    }
+    // Additional patterns to catch query craft related logs
+    if (/\[FAST\]/i.test(l) || /\[GEN\]/i.test(l) || /\[MICRO-HINT\]/i.test(l)) { 
+      mode = 'querycraft'; 
+      console.log(`[DEBUG] splitLogs: Found QUERYCRAFT pattern at line ${i}: "${l}"`);
+    }
+    // Heuristic: SQL lines go to querycraft
+    if (/^(SELECT|WITH|INSERT|UPDATE|DELETE)\b/i.test(l)) { 
+      mode = 'querycraft'; 
+      sections.querycraft += l + '\n'; 
+      console.log(`[DEBUG] splitLogs: Found SQL pattern at line ${i}, mode set to querycraft`);
+      continue; 
+    }
+    if (!mode) {
+      // Seed with memory until a clearer marker appears
+      sections.memory += line + '\n';
+    } else {
+      sections[mode] += line + '\n';
+    }
+  }
+
+  console.log('[DEBUG] splitLogs: Markers found - Memory:', memoryMarkerFound, 'QueryCraft:', querycraftMarkerFound, 'Presenter:', presenterMarkerFound);
+  console.log('[DEBUG] splitLogs: Section lengths - Memory:', sections.memory.length, 'QueryCraft:', sections.querycraft.length, 'Presenter:', sections.presenter.length);
+  console.log('[DEBUG] splitLogs: Memory section preview:', sections.memory.substring(0, 200));
+  console.log('[DEBUG] splitLogs: QueryCraft section preview:', sections.querycraft.substring(0, 200));
+  console.log('[DEBUG] splitLogs: Presenter section preview:', sections.presenter.substring(0, 200));
+
+  return sections;
+}
+
+function findSQL(logs) {
+  if (!logs) return '';
+  // Primary: explicit markers
+  const markerRe = /<<<SQL_REVISED_START>>>([\s\S]*?)<<<SQL_REVISED_END>>>/m;
+  let m = logs.match(markerRe);
+  if (m) return m[1].trim();
+  const markerRe2 = /<<<SQL_START>>>([\s\S]*?)<<<SQL_END>>>/m;
+  m = logs.match(markerRe2);
+  if (m) return m[1].trim();
+  // Fallback: first standalone SQL-ish block up to semicolon
+  const fallback = logs.match(/^(?:.*?)(SELECT|WITH)[^;]{0,4000};/im);
+  if (fallback) {
+    const startIdx = fallback.index;
+    if (startIdx != null) {
+      const slice = logs.slice(startIdx).match(/([\s\S]*?;)/);
+      if (slice) return slice[1].trim();
+    }
+  }
+  return '';
+}
+
+async function runQuery(q){
+  if (!q) return alert('Please enter a question');
+  answerEl.textContent = 'Thinking...';
+  setRunning(true);
+  timeEl.textContent = '';
+  sqlEl.textContent = '-- SQL will appear here --';
+  memoryEl.textContent = querycraftEl.textContent = presenterEl.textContent = '—';
+  
+  const startTime = Date.now();
+  
+  try {
+    const body = { question: q, mode: currentMode };
+    if (currentMode === 'ocr_qa' && currentDocId) body.doc_id = currentDocId;
+    const res = await fetch('/api/query', {
+      method: 'POST', headers: { 'Content-Type': 'application/json' },
+      body: JSON.stringify(body)
+    });
+    let j = null;
+    if (!res.ok) {
+      try { j = await res.json(); } catch(err){ const t = await res.text(); answerEl.textContent = 'Error: ' + t; setRunning(false); return; }
+      if (j.error === 'worker_timeout') { answerEl.textContent = 'Worker timeout: ' + (j.message || 'The worker did not respond in time.'); setRunning(false); return; }
+      answerEl.textContent = j.error || j.message || JSON.stringify(j);
+      if (j.trace) presenterEl.textContent = j.trace;
+      setRunning(false);
+      return;
+    }
+    j = await res.json();
+    
+    const elapsed = ((Date.now() - startTime) / 1000).toFixed(1);
+    
+    if (j.answer) {
+      // Render markdown/pipe tables as HTML tables for readability
+      if(isMarkdownTable(j.answer)){
+        const node = parseMarkdownTable(j.answer);
+        answerEl.innerHTML = '';
+        if(node) answerEl.appendChild(node);
+        else answerEl.textContent = j.answer;
+      } else {
+        // Non-table answers: preserve plain text
+        answerEl.textContent = j.answer;
+      }
+    } else answerEl.textContent = JSON.stringify(j, null, 2);
+    
+    // Prefer direct sql field from worker, else derive from logs
+    if (j.sql) {
+      const codeEl = document.getElementById('sql');
+      codeEl.textContent = j.sql;
+      if(window.Prism && Prism.highlightElement) Prism.highlightElement(codeEl);
+    }
+    
+    if (j.logs) {
+      console.log('[DEBUG] runQuery: Processing logs of length:', j.logs.length);
+      const sections = splitLogs(j.logs);
+      console.log('[DEBUG] runQuery: Sections received:', {
+        memory: sections.memory.length,
+        querycraft: sections.querycraft.length,
+        presenter: sections.presenter.length
+      });
+      
+      const memoryContent = sections.memory.trim() || '—';
+      const querycraftContent = sections.querycraft.trim() || '—';
+      const presenterContent = sections.presenter.trim() || '';
+      
+      console.log('[DEBUG] runQuery: Setting memory content length:', memoryContent.length);
+      console.log('[DEBUG] runQuery: Setting querycraft content length:', querycraftContent.length);
+      console.log('[DEBUG] runQuery: Setting presenter content length:', presenterContent.length);
+      
+      memoryEl.textContent = memoryContent;
+      querycraftEl.textContent = querycraftContent;
+      
+      if(presenterContent){ 
+        console.log('[DEBUG] runQuery: Rendering presenter trace');
+        const node = renderPresenterTrace(presenterContent); 
+        presenterEl.innerHTML = ''; 
+        presenterEl.appendChild(node); 
+      } else { 
+        console.log('[DEBUG] runQuery: No presenter content, setting dash');
+        presenterEl.textContent = '—'; 
+      }
+      if (!j.sql) {
+        const sql = findSQL(j.logs) || '';
+        if (sql) { 
+          const codeEl = document.getElementById('sql'); 
+          codeEl.textContent = sql; 
+          if(window.Prism && Prism.highlightElement) Prism.highlightElement(codeEl); 
+        }
+      }
+    } else {
+      console.log('[DEBUG] runQuery: No logs in response');
+    }
+    
+    statusEl.textContent = 'Done';
+    timeEl.textContent = `${elapsed}s`;
+    setRunning(false);
+  } catch (e) {
+    answerEl.textContent = 'Error: ' + String(e);
+    setRunning(false);
+  }
+}
+
+askBtn.onclick = () => runQuery(questionEl.value.trim());
+
+recalcBtn.onclick = () => {
+  const q = questionEl.value.trim();
+  if (!q) return alert('Please enter a question');
+  const normalized = q.replace(/^(?:re-execute\s+)+/i, '');
+  const prefixed = 're-execute ' + normalized;
+  // do not modify the visible textarea; send the prefixed query directly
+  runQuery(prefixed);
+};
+
+// Enter key support
+questionEl.addEventListener('keydown', (e) => {
+  if (e.key === 'Enter' && (e.ctrlKey || e.metaKey)) {
+    e.preventDefault();
+    runQuery(questionEl.value.trim());
+  }
+});
+

web/static/index.html

@@ -0,0 +1,996 @@
+<!doctype html>
+<html lang="en">
+
+<head>
+    <meta charset="utf-8" />
+    <meta name="viewport" content="width=device-width, initial-scale=1" />
+    <title>QueryLens - Transform natural language into SQL</title>
+    <link rel="icon"
+        href="data:image/svg+xml;base64,PHN2ZyB3aWR0aD0iMjQiIGhlaWdodD0iMjQiIHZpZXdCb3g9IjAgMCAyNCAyNCIgZmlsbD0ibm9uZSIgeG1sbnM9Imh0dHA6Ly93d3cudzMub3JnLzIwMDAvc3ZnIj4KPGNpcmNsZSBjeD0iMTIiIGN5PSIxMiIgcj0iMTAiIGZpbGw9IiMwRjYyRkUiLz4KPHN2ZyB3aWR0aD0iMTYiIGhlaWdodD0iMTYiIHZpZXdCb3g9IjAgMCAyNCAyNCIgZmlsbD0ibm9uZSIgeD0iNCIgeT0iNCIgY2xhc3M9InF1ZXJ5LWljb24iPgo8cGF0aCBkPSJNMTIgMlY3TTEyIDdMMTUgM001IDkuNUg5LjVNOS41IDkuNUw2IDZNMTkgMTQuNUgxNC41TTE0LjUgMTQuNUwxOCAxOE0xMiAxN1YyMk0xMiAyMkw5IDE5IiBzdHJva2U9IndoaXRlIiBzdHJva2Utd2lkdGg9IjIiIHN0cm9rZS1saW5lY2FwPSJyb3VuZCIgc3Ryb2tlLWxpbmVqb2luPSJyb3VuZCIvPgo8L3N2Zz4KPC9zdmc+">
+
+    <!-- Fonts -->
+    <link rel="preconnect" href="https://fonts.googleapis.com">
+    <link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
+    <link href="https://fonts.googleapis.com/css2?family=Inter:wght@300;400;500;600;700;800&display=swap"
+        rel="stylesheet">
+
+    <!-- Icons -->
+    <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/lucide@0.292.0/dist/umd/lucide.min.css">
+
+    <style>
+        :root {
+            /* Modern Color Palette */
+            --primary: #0F62FE;
+            --primary-dark: #0353E9;
+            --primary-light: #4589FF;
+            --secondary: #8A3FFC;
+            --accent: #FF6B35;
+
+            /* Neutrals - Enhanced for better contrast */
+            --gray-50: #F8FAFC;
+            --gray-100: #F1F5F9;
+            --gray-200: #E2E8F0;
+            --gray-300: #CBD5E1;
+            --gray-400: #94A3B8;
+            --gray-500: #64748B;
+            --gray-600: #475569;
+            --gray-700: #334155;
+            --gray-800: #1E293B;
+            --gray-900: #0F172A;
+
+            /* Semantic */
+            --success: #10B981;
+            --warning: #F59E0B;
+            --error: #EF4444;
+
+            /* Surfaces - Improved contrast */
+            --bg-primary: #FAFBFC;
+            /* Slightly off-white for better contrast */
+            --bg-secondary: #F4F6F8;
+            /* Darker secondary background */
+            --bg-tertiary: #E8EDF3;
+            /* More distinct tertiary */
+            --card-bg: #FFFFFF;
+            /* Pure white cards for contrast */
+            --border: #D1D9E0;
+            /* Darker borders for definition */
+            --border-light: #E5EAF0;
+            /* Subtle borders */
+
+            /* Typography - Enhanced contrast */
+            --text-primary: #1A202C;
+            /* Darker primary text */
+            --text-secondary: #4A5568;
+            /* Stronger secondary text */
+            --text-tertiary: #718096;
+            /* More readable tertiary text */
+
+            /* Spacing */
+            --space-1: 0.25rem;
+            --space-2: 0.5rem;
+            --space-3: 0.75rem;
+            --space-4: 1rem;
+            --space-5: 1.25rem;
+            --space-6: 1.5rem;
+            --space-8: 2rem;
+            --space-10: 2.5rem;
+            --space-12: 3rem;
+            --space-16: 4rem;
+
+            /* Shadows */
+            --shadow-sm: 0 1px 2px 0 rgb(0 0 0 / 0.05);
+            --shadow-md: 0 4px 6px -1px rgb(0 0 0 / 0.1), 0 2px 4px -2px rgb(0 0 0 / 0.1);
+            --shadow-lg: 0 10px 15px -3px rgb(0 0 0 / 0.1), 0 4px 6px -4px rgb(0 0 0 / 0.1);
+            --shadow-xl: 0 20px 25px -5px rgb(0 0 0 / 0.1), 0 8px 10px -6px rgb(0 0 0 / 0.1);
+
+            /* Border Radius */
+            --radius-sm: 0.375rem;
+            --radius-md: 0.5rem;
+            --radius-lg: 0.75rem;
+            --radius-xl: 1rem;
+
+            /* Transitions */
+            --transition-fast: 150ms ease;
+            --transition-normal: 250ms ease;
+            --transition-slow: 350ms ease;
+        }
+
+        /* Reset & Base Styles */
+        *,
+        *::before,
+        *::after {
+            box-sizing: border-box;
+        }
+
+        html,
+        body {
+            height: 100%;
+            margin: 0;
+            font-family: 'Inter', -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, sans-serif;
+            background: var(--bg-secondary);
+            color: var(--text-primary);
+            -webkit-font-smoothing: antialiased;
+            -moz-osx-font-smoothing: grayscale;
+            font-size: 16px;
+            line-height: 1.6;
+        }
+
+        /* Layout */
+        .app-container {
+            min-height: 100vh;
+            display: flex;
+            flex-direction: column;
+        }
+
+        .main-content {
+            flex: 1;
+            max-width: 1400px;
+            margin: 0 auto;
+            padding: var(--space-6) var(--space-4);
+            width: 100%;
+        }
+
+        /* Header */
+        .header {
+            background: var(--card-bg);
+            border-bottom: 1px solid var(--border);
+            box-shadow: var(--shadow-sm);
+            position: sticky;
+            top: 0;
+            z-index: 50;
+        }
+
+        .header-content {
+            max-width: 1400px;
+            margin: 0 auto;
+            padding: var(--space-4) var(--space-4);
+            display: flex;
+            align-items: center;
+            gap: var(--space-4);
+        }
+
+        .brand {
+            display: flex;
+            align-items: center;
+            gap: var(--space-3);
+        }
+
+        .brand-icon {
+            width: 48px;
+            height: 48px;
+            background: linear-gradient(135deg, var(--primary) 0%, var(--primary-dark) 100%);
+            border-radius: var(--radius-lg);
+            display: flex;
+            align-items: center;
+            justify-content: center;
+            box-shadow: var(--shadow-md);
+            position: relative;
+            overflow: hidden;
+        }
+
+        .brand-icon::before {
+            content: '';
+            position: absolute;
+            top: 0;
+            left: 0;
+            right: 0;
+            bottom: 0;
+            background: linear-gradient(45deg, transparent 30%, rgba(255, 255, 255, 0.1) 50%, transparent 70%);
+            animation: shimmer 3s infinite;
+        }
+
+        @keyframes shimmer {
+            0% {
+                transform: translateX(-100%);
+            }
+
+            100% {
+                transform: translateX(100%);
+            }
+        }
+
+        .brand-icon svg {
+            width: 24px;
+            height: 24px;
+            color: white;
+            z-index: 1;
+            position: relative;
+        }
+
+        .brand-text h1 {
+            margin: 0;
+            font-size: 1.875rem;
+            font-weight: 800;
+            background: linear-gradient(135deg, var(--text-primary) 0%, var(--primary) 100%);
+            -webkit-background-clip: text;
+            -webkit-text-fill-color: transparent;
+            background-clip: text;
+            letter-spacing: -0.025em;
+        }
+
+        .brand-text p {
+            margin: 0;
+            color: var(--text-secondary);
+            font-size: 0.875rem;
+            font-weight: 500;
+        }
+
+        /* Query Section */
+        .query-section {
+            background: var(--card-bg);
+            border-radius: var(--radius-xl);
+            box-shadow: var(--shadow-lg);
+            border: 1px solid var(--border);
+            margin-bottom: var(--space-8);
+            overflow: hidden;
+        }
+
+        .query-header {
+            padding: var(--space-6) var(--space-6) 0;
+        }
+
+        .query-title {
+            margin: 0 0 var(--space-2);
+            font-size: 1.125rem;
+            font-weight: 600;
+            color: var(--text-primary);
+        }
+
+        .query-body {
+            padding: 0 var(--space-6) var(--space-6);
+        }
+
+        .textarea-container {
+            position: relative;
+            margin-bottom: var(--space-4);
+        }
+
+        .query-textarea {
+            width: 100%;
+            min-height: 120px;
+            padding: var(--space-4);
+            border: 2px solid var(--border);
+            border-radius: var(--radius-lg);
+            font-size: 1rem;
+            font-family: inherit;
+            line-height: 1.6;
+            resize: vertical;
+            background: var(--bg-primary);
+            color: var(--text-primary);
+            transition: var(--transition-normal);
+        }
+
+        .query-textarea:focus {
+            outline: none;
+            border-color: var(--primary);
+            box-shadow: 0 0 0 3px rgba(15, 98, 254, 0.1);
+        }
+
+        .query-textarea::placeholder {
+            color: var(--text-tertiary);
+        }
+
+        .query-controls {
+            display: flex;
+            align-items: center;
+            gap: var(--space-3);
+            flex-wrap: wrap;
+        }
+
+        .mode-selector {
+            display: flex;
+            align-items: center;
+            gap: var(--space-2);
+        }
+
+        .mode-selector label {
+            font-size: 0.875rem;
+            font-weight: 500;
+            color: var(--text-secondary);
+        }
+
+        .mode-select {
+            padding: var(--space-2) var(--space-3);
+            border: 1px solid var(--border);
+            border-radius: var(--radius-md);
+            font-size: 0.875rem;
+            background: var(--bg-primary);
+            color: var(--text-primary);
+            min-width: 120px;
+        }
+
+        /* Buttons */
+        .btn {
+            display: inline-flex;
+            align-items: center;
+            gap: var(--space-2);
+            padding: var(--space-3) var(--space-4);
+            border-radius: var(--radius-md);
+            font-size: 0.875rem;
+            font-weight: 600;
+            border: none;
+            cursor: pointer;
+            transition: var(--transition-normal);
+            text-decoration: none;
+            position: relative;
+            overflow: hidden;
+        }
+
+        .btn:disabled {
+            opacity: 0.5;
+            cursor: not-allowed;
+        }
+
+        .btn-primary {
+            background: var(--primary);
+            color: white;
+            box-shadow: var(--shadow-sm);
+        }
+
+        .btn-primary:hover:not(:disabled) {
+            background: var(--primary-dark);
+            transform: translateY(-1px);
+            box-shadow: var(--shadow-md);
+        }
+
+        .btn-secondary {
+            background: var(--bg-primary);
+            color: var(--text-primary);
+            border: 1px solid var(--border);
+            box-shadow: var(--shadow-sm);
+        }
+
+        .btn-secondary:hover:not(:disabled) {
+            background: var(--bg-tertiary);
+            transform: translateY(-1px);
+        }
+
+        .upload-controls {
+            display: flex;
+            align-items: center;
+            gap: var(--space-3);
+        }
+
+        .status-area {
+            margin-left: auto;
+            display: flex;
+            align-items: center;
+            gap: var(--space-3);
+        }
+
+        /* Spinner */
+        .spinner {
+            width: 24px;
+            height: 24px;
+            border: 3px solid var(--border);
+            border-top-color: var(--primary);
+            border-radius: 50%;
+            animation: spin 1s linear infinite;
+            display: inline-block;
+        }
+
+        .spinner.hidden {
+            display: none !important;
+        }
+
+        @keyframes spin {
+            to {
+                transform: rotate(360deg);
+            }
+        }
+
+        .status-text {
+            font-size: 0.875rem;
+            color: var(--text-secondary);
+        }
+
+        /* Results Grid */
+        .results-grid {
+            display: grid;
+            grid-template-columns: 1fr;
+            gap: var(--space-6);
+        }
+
+        @media (min-width: 1024px) {
+            .results-grid {
+                grid-template-columns: 2fr 1fr;
+                gap: var(--space-6);
+            }
+
+            /* CRITICAL FIX: Prevent grid blowout from wide tables */
+            .results-grid>.main-results {
+                min-width: 0;
+                /* Allow grid cell to shrink below content size */
+                overflow: hidden;
+                /* Contain overflowing content */
+            }
+
+            .results-grid>.agent-panels {
+                min-width: 0;
+                /* Prevent agent panels from being pushed off-screen */
+            }
+        }
+
+        /* Cards */
+        .card {
+            background: var(--card-bg);
+            border-radius: var(--radius-xl);
+            border: 1px solid var(--border);
+            box-shadow: var(--shadow-md);
+            overflow: hidden;
+        }
+
+        .card-header {
+            padding: var(--space-4) var(--space-6);
+            border-bottom: 1px solid var(--border-light);
+            background: var(--bg-tertiary);
+            display: flex;
+            align-items: center;
+            justify-content: space-between;
+        }
+
+        .card-title {
+            margin: 0;
+            font-size: 1rem;
+            font-weight: 600;
+            color: var(--text-primary);
+            display: flex;
+            align-items: center;
+            gap: var(--space-2);
+        }
+
+        .card-content {
+            padding: var(--space-6);
+        }
+
+        .answer-content {
+            font-family: 'SF Mono', Monaco, 'Cascadia Code', 'Roboto Mono', Consolas, 'Courier New', monospace;
+            background: var(--bg-secondary);
+            border-radius: var(--radius-lg);
+            padding: var(--space-4);
+            white-space: pre-wrap;
+            color: var(--text-primary);
+            font-size: 0.875rem;
+            line-height: 1.6;
+            min-height: 120px;
+            border: 1px solid var(--border-light);
+        }
+
+        .sql-content {
+            background: var(--gray-900);
+            color: #E2E8F0;
+            border-radius: var(--radius-lg);
+            padding: var(--space-4);
+            font-family: 'SF Mono', Monaco, 'Cascadia Code', 'Roboto Mono', Consolas, 'Courier New', monospace;
+            font-size: 0.875rem;
+            line-height: 1.6;
+            overflow-x: auto;
+            max-height: 300px;
+            overflow-y: auto;
+        }
+
+        /* Agent Panels */
+        .agent-panels {
+            display: grid;
+            gap: var(--space-4);
+        }
+
+        .panel {
+            border: 1px solid var(--border-light);
+            border-radius: var(--radius-lg);
+            overflow: hidden;
+        }
+
+        .panel-header {
+            padding: var(--space-3) var(--space-4);
+            background: var(--bg-tertiary);
+            border-bottom: 1px solid var(--border-light);
+            cursor: pointer;
+            user-select: none;
+            display: flex;
+            align-items: center;
+            justify-content: space-between;
+            transition: var(--transition-fast);
+        }
+
+        .panel-header:hover {
+            background: var(--bg-secondary);
+        }
+
+        .panel-title {
+            font-size: 0.875rem;
+            font-weight: 600;
+            color: var(--text-primary);
+        }
+
+        .panel-content {
+            padding: var(--space-4);
+            font-family: 'SF Mono', Monaco, 'Cascadia Code', 'Roboto Mono', Consolas, 'Courier New', monospace;
+            font-size: 0.8rem;
+            line-height: 1.5;
+            color: var(--text-secondary);
+            white-space: pre-wrap;
+            max-height: 300px;
+            overflow-y: auto;
+            background: var(--bg-primary);
+        }
+
+        /* Tables */
+        .table-wrapper {
+            position: relative;
+            margin: var(--space-4) 0;
+            border-radius: var(--radius-lg);
+            box-shadow: var(--shadow-lg);
+            background: var(--card-bg);
+            border: 1px solid var(--border);
+            overflow: hidden;
+            max-width: 100%;
+            /* Prevent table from breaking grid layout */
+        }
+
+        .table-controls {
+            display: flex;
+            align-items: center;
+            justify-content: space-between;
+            padding: var(--space-3) var(--space-4);
+            background: var(--bg-tertiary);
+            border-bottom: 1px solid var(--border);
+            gap: var(--space-3);
+        }
+
+        .table-controls-left {
+            display: flex;
+            align-items: center;
+            gap: var(--space-3);
+        }
+
+        .table-controls-right {
+            display: flex;
+            align-items: center;
+            gap: var(--space-2);
+        }
+
+        .table-info {
+            font-size: 0.875rem;
+            color: var(--text-secondary);
+            font-weight: 500;
+        }
+
+        .table-resize-controls {
+            display: flex;
+            align-items: center;
+            gap: var(--space-2);
+        }
+
+        .table-resize-btn {
+            padding: var(--space-1) var(--space-2);
+            border: 1px solid var(--border);
+            border-radius: var(--radius-sm);
+            background: var(--bg-primary);
+            color: var(--text-secondary);
+            font-size: 0.75rem;
+            cursor: pointer;
+            transition: var(--transition-normal);
+        }
+
+        .table-resize-btn:hover {
+            background: var(--bg-secondary);
+            color: var(--text-primary);
+        }
+
+        .table-resize-btn.active {
+            background: var(--primary);
+            color: white;
+            border-color: var(--primary);
+        }
+
+        .table-container {
+            overflow: auto;
+            max-height: 70vh;
+            max-width: 100%;
+            position: relative;
+            border: 1px solid var(--border);
+            border-radius: var(--radius-lg);
+        }
+
+        .answer-table {
+            width: 100%;
+            border-collapse: separate;
+            border-spacing: 0;
+            font-family: 'Inter', -apple-system, BlinkMacSystemFont, sans-serif;
+            font-size: 0.875rem;
+            background: var(--card-bg);
+            table-layout: auto;
+            min-width: 100%;
+        }
+
+        .answer-table.compact {
+            table-layout: fixed;
+        }
+
+        .answer-table.comfortable {
+            table-layout: auto;
+        }
+
+        .answer-table.spacious {
+            table-layout: auto;
+            font-size: 1rem;
+        }
+
+        .answer-table th,
+        .answer-table td {
+            padding: var(--space-3) var(--space-4);
+            text-align: left;
+            border-bottom: 1px solid var(--border-light);
+            border-right: 1px solid var(--border-light);
+            vertical-align: top;
+            position: relative;
+        }
+
+        .answer-table.compact th,
+        .answer-table.compact td {
+            padding: var(--space-2) var(--space-3);
+            max-width: 120px;
+            overflow: hidden;
+            text-overflow: ellipsis;
+            white-space: nowrap;
+        }
+
+        .answer-table.comfortable th,
+        .answer-table.comfortable td {
+            padding: var(--space-3) var(--space-4);
+            max-width: 200px;
+            word-wrap: break-word;
+            white-space: normal;
+        }
+
+        .answer-table.spacious th,
+        .answer-table.spacious td {
+            padding: var(--space-4) var(--space-6);
+            max-width: 300px;
+            word-wrap: break-word;
+            white-space: normal;
+            overflow-wrap: break-word;
+        }
+
+        .answer-table th {
+            background: #F1F5F9;
+            /* Stronger header background */
+            font-weight: 600;
+            color: #1A202C;
+            /* Darker header text */
+            position: sticky;
+            top: 0;
+            z-index: 10;
+            border-top: none;
+            border-bottom: 2px solid #D1D9E0;
+            /* Stronger header border */
+        }
+
+        .answer-table th:last-child,
+        .answer-table td:last-child {
+            border-right: none;
+        }
+
+        .answer-table tbody tr:hover {
+            background: #E8F4FD;
+            /* Subtle blue hover */
+        }
+
+        .answer-table tbody tr:nth-child(even) {
+            background: #F7F9FC;
+            /* More visible alternating rows */
+        }
+
+        .answer-table tbody tr:nth-child(even):hover {
+            background: #E8F4FD;
+            /* Consistent hover color */
+        }
+
+        /* Column resizer */
+        .column-resizer {
+            position: absolute;
+            right: 0;
+            top: 0;
+            bottom: 0;
+            width: 4px;
+            cursor: col-resize;
+            background: transparent;
+            transition: background-color 0.2s;
+        }
+
+        .column-resizer:hover {
+            background: var(--primary);
+        }
+
+        /* Responsive adjustments */
+        @media (max-width: 768px) {
+            .table-controls {
+                flex-direction: column;
+                align-items: stretch;
+                gap: var(--space-2);
+            }
+
+            .table-controls-left,
+            .table-controls-right {
+                justify-content: space-between;
+            }
+
+            .answer-table th,
+            .answer-table td {
+                padding: var(--space-2) var(--space-3);
+                font-size: 0.8rem;
+            }
+        }
+
+        /* Footer */
+        .footer {
+            margin-top: var(--space-12);
+            padding: var(--space-8) 0;
+            text-align: center;
+            color: var(--text-tertiary);
+            font-size: 0.875rem;
+            border-top: 1px solid var(--border-light);
+        }
+
+        /* Responsive */
+        @media (max-width: 768px) {
+            .main-content {
+                padding: var(--space-4) var(--space-3);
+            }
+
+            .header-content {
+                padding: var(--space-3);
+            }
+
+            .query-controls {
+                flex-direction: column;
+                align-items: stretch;
+            }
+
+            .status-area {
+                margin-left: 0;
+                margin-top: var(--space-3);
+            }
+        }
+
+        /* Utility Classes */
+        .hidden {
+            display: none !important;
+        }
+
+        .text-success {
+            color: var(--success) !important;
+        }
+
+        .text-warning {
+            color: var(--warning) !important;
+        }
+
+        .text-error {
+            color: var(--error) !important;
+        }
+
+        /* Upload Controls - Show/Hide */
+        .upload-controls.hidden {
+            display: none !important;
+        }
+
+        .upload-controls {
+            display: flex !important;
+            align-items: center;
+            gap: var(--space-3);
+        }
+    </style>
+
+    <!-- Prism for SQL highlighting -->
+    <link href="https://cdnjs.cloudflare.com/ajax/libs/prism/1.29.0/themes/prism-tomorrow.min.css" rel="stylesheet" />
+</head>
+
+<body>
+    <div class="app-container">
+        <!-- Header -->
+        <header class="header">
+            <div class="header-content">
+                <div class="brand">
+                    <div class="brand-icon">
+                        <svg width="24" height="24" viewBox="0 0 24 24" fill="none" xmlns="http://www.w3.org/2000/svg">
+                            <path
+                                d="M12 2V7M12 7L15 3M5 9.5H9.5M9.5 9.5L6 6M19 14.5H14.5M14.5 14.5L18 18M12 17V22M12 22L9 19"
+                                stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" />
+                        </svg>
+                    </div>
+                    <div class="brand-text">
+                        <h1>QueryLens</h1>
+                        <p>Transform natural language into SQL, and documents into insight.</p>
+                    </div>
+                </div>
+            </div>
+        </header>
+
+        <!-- Main Content -->
+        <main class="main-content">
+            <!-- Query Section -->
+            <section class="query-section">
+                <div class="query-header">
+                    <h2 class="query-title">Ask a Question</h2>
+                </div>
+                <div class="query-body">
+                    <div class="textarea-container">
+                        <textarea id="question" class="query-textarea"
+                            placeholder="e.g. Which department has the highest average salary?"></textarea>
+                    </div>
+
+                    <div class="query-controls">
+                        <div class="mode-selector">
+                            <label for="modeToggle">Mode:</label>
+                            <select id="modeToggle" class="mode-select">
+                                <option value="text2sql">Text-to-SQL</option>
+                                <option value="ocr_qa">OCR Q&A</option>
+                            </select>
+                        </div>
+
+                        <button id="ask" class="btn btn-primary">
+                            <svg width="16" height="16" viewBox="0 0 24 24" fill="none" stroke="currentColor"
+                                stroke-width="2">
+                                <path d="M12 19l7-7 3 3-7 7-3-3z" />
+                                <path d="M18 13l-1.5-7.5L2 2l3.5 14.5L13 18l5-5z" />
+                            </svg>
+                            Ask
+                        </button>
+
+                        <button id="recalc" class="btn btn-secondary">
+                            <svg width="16" height="16" viewBox="0 0 24 24" fill="none" stroke="currentColor"
+                                stroke-width="2">
+                                <path d="M1 4v6h6" />
+                                <path d="M23 20v-6h-6" />
+                                <path d="M20.49 9A9 9 0 0 0 5.64 5.64L1 10m22 4l-4.64 4.36A9 9 0 0 1 3.51 15" />
+                            </svg>
+                            Re-execute
+                        </button>
+
+                        <div id="uploadControls" class="upload-controls hidden">
+                            <button id="uploadBtn" class="btn btn-secondary">
+                                <svg width="16" height="16" viewBox="0 0 24 24" fill="none" stroke="currentColor"
+                                    stroke-width="2">
+                                    <path d="M21 15v4a2 2 0 0 1-2 2H5a2 2 0 0 1-2-2v-4" />
+                                    <polyline points="17,8 12,3 7,8" />
+                                    <line x1="12" y1="3" x2="12" y2="15" />
+                                </svg>
+                                Upload Image
+                            </button>
+                            <span id="uploadStatus" class="status-text">No upload</span>
+                        </div>
+
+                        <div class="status-area">
+                            <div id="spinner" class="spinner hidden"></div>
+                            <span id="spinnerText" class="status-text hidden">Processing...</span>
+                            <span id="status" class="status-text">Ready</span>
+                            <span id="time" class="status-text"></span>
+                        </div>
+                    </div>
+                </div>
+            </section>
+
+            <!-- Results Grid -->
+            <div class="results-grid">
+                <!-- Main Results Column -->
+                <div class="main-results">
+                    <!-- Answer Card -->
+                    <div class="card">
+                        <div class="card-header">
+                            <h3 class="card-title">
+                                <svg width="16" height="16" viewBox="0 0 24 24" fill="none" stroke="currentColor"
+                                    stroke-width="2">
+                                    <path d="M9 11l3 3L22 4" />
+                                    <path d="M21 12v7a2 2 0 0 1-2 2H5a2 2 0 0 1-2-2V5a2 2 0 0 1 2-2h11" />
+                                </svg>
+                                Answer
+                            </h3>
+                        </div>
+                        <div class="card-content">
+                            <div id="answer" class="answer-content">No answer yet.</div>
+                        </div>
+                    </div>
+
+                    <!-- SQL Card -->
+                    <div class="card" style="margin-top: var(--space-6);">
+                        <details id="sql-details" open>
+                            <summary class="card-header" style="cursor: pointer;">
+                                <h3 class="card-title">
+                                    <svg width="16" height="16" viewBox="0 0 24 24" fill="none" stroke="currentColor"
+                                        stroke-width="2">
+                                        <polyline points="16,18 22,12 16,6" />
+                                        <polyline points="8,6 2,12 8,18" />
+                                    </svg>
+                                    Generated SQL
+                                </h3>
+                            </summary>
+                            <div class="card-content">
+                                <pre
+                                    class="sql-content"><code id="sql" class="language-sql">-- SQL will appear here after the agent crafts a query --</code></pre>
+                            </div>
+                        </details>
+                    </div>
+                </div>
+
+                <!-- Agent Panels Column -->
+                <div class="agent-panels">
+                    <div class="panel">
+                        <details open>
+                            <summary class="panel-header">
+                                <span class="panel-title">
+                                    <svg width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor"
+                                        stroke-width="2" style="margin-right: 0.5rem;">
+                                        <path
+                                            d="M9 19c-5 1.5-5-2.5-7-3m14 6v-3.87a3.37 3.37 0 0 0-.94-2.61c3.14-.35 6.44-1.54 6.44-7A5.44 5.44 0 0 0 20 4.77 5.07 5.07 0 0 0 19.91 1S18.73.65 16 2.48a13.38 13.38 0 0 0-7 0C6.27.65 5.09 1 5.09 1A5.07 5.07 0 0 0 5 4.77a5.44 5.44 0 0 0-1.5 3.78c0 5.42 3.3 6.61 6.44 7A3.37 3.37 0 0 0 9 18.13V22" />
+                                    </svg>
+                                    Memory Agent
+                                </span>
+                                <svg width="16" height="16" viewBox="0 0 24 24" fill="none" stroke="currentColor"
+                                    stroke-width="2">
+                                    <polyline points="6,9 12,15 18,9" />
+                                </svg>
+                            </summary>
+                            <div id="memory" class="panel-content">—</div>
+                        </details>
+                    </div>
+
+                    <div class="panel">
+                        <details open>
+                            <summary class="panel-header">
+                                <span class="panel-title">
+                                    <svg width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor"
+                                        stroke-width="2" style="margin-right: 0.5rem;">
+                                        <path d="M12 20h9" />
+                                        <path d="M16.5 3.5a2.121 2.121 0 0 1 3 3L7 19l-4 1 1-4L16.5 3.5z" />
+                                    </svg>
+                                    Query Craft Agent
+                                </span>
+                                <svg width="16" height="16" viewBox="0 0 24 24" fill="none" stroke="currentColor"
+                                    stroke-width="2">
+                                    <polyline points="6,9 12,15 18,9" />
+                                </svg>
+                            </summary>
+                            <div id="querycraft" class="panel-content">—</div>
+                        </details>
+                    </div>
+
+                    <div class="panel">
+                        <details open>
+                            <summary class="panel-header">
+                                <span class="panel-title">
+                                    <svg width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor"
+                                        stroke-width="2" style="margin-right: 0.5rem;">
+                                        <path d="M14 2H6a2 2 0 0 0-2 2v16a2 2 0 0 0 2 2h12a2 2 0 0 0 2-2V8z" />
+                                        <polyline points="14,2 14,8 20,8" />
+                                        <line x1="16" y1="13" x2="8" y2="13" />
+                                        <line x1="16" y1="17" x2="8" y2="17" />
+                                        <polyline points="10,9 9,9 8,9" />
+                                    </svg>
+                                    Result Presenter Agent
+                                </span>
+                                <svg width="16" height="16" viewBox="0 0 24 24" fill="none" stroke="currentColor"
+                                    stroke-width="2">
+                                    <polyline points="6,9 12,15 18,9" />
+                                </svg>
+                            </summary>
+                            <div id="presenter" class="panel-content">—</div>
+                        </details>
+                    </div>
+                </div>
+            </div>
+        </main>
+
+        <!-- Footer -->
+        <footer class="footer">
+            <p>QueryLens — Powered by advanced AI agents for intelligent data analysis</p>
+        </footer>
+    </div>
+
+    <script src="/static/app.js"></script>
+    <script src="https://cdnjs.cloudflare.com/ajax/libs/prism/1.29.0/prism.min.js"></script>
+    <script src="https://cdnjs.cloudflare.com/ajax/libs/prism/1.29.0/components/prism-sql.min.js"></script>
+</body>
+
+</html>

web/static/table-demo.html

@@ -0,0 +1,338 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+    <meta charset="UTF-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+    <title>Table Demo</title>
+    <style>
+        /* Copy the same CSS variables and table styles from the main app */
+        :root {
+            --primary: #0f62fe;
+            --primary-hover: #0353e9;
+            --bg-primary: #FAFBFC;
+            --bg-secondary: #F4F6F8;
+            --bg-tertiary: #E8EDF3;
+            --card-bg: #ffffff;
+            --text-primary: #1A202C;
+            --text-secondary: #4A5568;
+            --text-tertiary: #718096;
+            --border: #D1D9E0;
+            --border-light: #E5EAF0;
+            --shadow-sm: 0 1px 2px 0 rgba(0, 0, 0, 0.05);
+            --shadow-lg: 0 10px 15px -3px rgba(0, 0, 0, 0.1), 0 4px 6px -2px rgba(0, 0, 0, 0.05);
+            --radius-sm: 0.25rem;
+            --radius-md: 0.375rem;
+            --radius-lg: 0.5rem;
+            --radius-xl: 0.75rem;
+            --space-1: 0.25rem;
+            --space-2: 0.5rem;
+            --space-3: 0.75rem;
+            --space-4: 1rem;
+            --space-6: 1.5rem;
+            --space-8: 2rem;
+            --space-12: 3rem;
+            --transition-normal: all 0.2s ease-in-out;
+        }
+
+        body {
+            font-family: 'Inter', -apple-system, BlinkMacSystemFont, sans-serif;
+            background: var(--bg-secondary);
+            color: var(--text-primary);
+            margin: 2rem;
+        }
+
+        .table-wrapper {
+            position: relative;
+            margin: var(--space-4) 0;
+            border-radius: var(--radius-lg);
+            box-shadow: var(--shadow-lg);
+            background: var(--card-bg);
+            border: 1px solid var(--border);
+            overflow: hidden;
+        }
+
+        .table-controls {
+            display: flex;
+            align-items: center;
+            justify-content: space-between;
+            padding: var(--space-3) var(--space-4);
+            background: var(--bg-tertiary);
+            border-bottom: 1px solid var(--border);
+            gap: var(--space-3);
+        }
+
+        .table-controls-left {
+            display: flex;
+            align-items: center;
+            gap: var(--space-3);
+        }
+
+        .table-controls-right {
+            display: flex;
+            align-items: center;
+            gap: var(--space-2);
+        }
+
+        .table-info {
+            font-size: 0.875rem;
+            color: var(--text-secondary);
+            font-weight: 500;
+        }
+
+        .table-resize-controls {
+            display: flex;
+            align-items: center;
+            gap: var(--space-2);
+        }
+
+        .table-resize-btn {
+            padding: var(--space-1) var(--space-2);
+            border: 1px solid var(--border);
+            border-radius: var(--radius-sm);
+            background: var(--bg-primary);
+            color: var(--text-secondary);
+            font-size: 0.75rem;
+            cursor: pointer;
+            transition: var(--transition-normal);
+        }
+
+        .table-resize-btn:hover {
+            background: var(--bg-secondary);
+            color: var(--text-primary);
+        }
+
+        .table-resize-btn.active {
+            background: var(--primary);
+            color: white;
+            border-color: var(--primary);
+        }
+
+        .table-container {
+            overflow: auto;
+            max-height: 70vh;
+            position: relative;
+        }
+
+        .answer-table {
+            width: 100%;
+            border-collapse: separate;
+            border-spacing: 0;
+            font-family: 'Inter', -apple-system, BlinkMacSystemFont, sans-serif;
+            font-size: 0.875rem;
+            background: var(--card-bg);
+            table-layout: auto;
+            min-width: 100%;
+        }
+
+        .answer-table.compact {
+            table-layout: fixed;
+        }
+
+        .answer-table.comfortable {
+            table-layout: auto;
+        }
+
+        .answer-table.spacious {
+            table-layout: auto;
+            font-size: 1rem;
+        }
+
+        .answer-table th,
+        .answer-table td {
+            padding: var(--space-3) var(--space-4);
+            text-align: left;
+            border-bottom: 1px solid var(--border-light);
+            border-right: 1px solid var(--border-light);
+            vertical-align: top;
+            position: relative;
+        }
+
+        .answer-table.compact th,
+        .answer-table.compact td {
+            padding: var(--space-2) var(--space-3);
+            max-width: 120px;
+            overflow: hidden;
+            text-overflow: ellipsis;
+            white-space: nowrap;
+        }
+
+        .answer-table.comfortable th,
+        .answer-table.comfortable td {
+            padding: var(--space-3) var(--space-4);
+            max-width: 200px;
+            word-wrap: break-word;
+            white-space: normal;
+        }
+
+        .answer-table.spacious th,
+        .answer-table.spacious td {
+            padding: var(--space-4) var(--space-6);
+            max-width: none;
+            word-wrap: break-word;
+            white-space: normal;
+        }
+
+        .answer-table th {
+            background: #F1F5F9;
+            font-weight: 600;
+            color: #1A202C;
+            position: sticky;
+            top: 0;
+            z-index: 10;
+            border-top: none;
+            border-bottom: 2px solid #D1D9E0;
+        }
+
+        .answer-table th:last-child,
+        .answer-table td:last-child {
+            border-right: none;
+        }
+
+        .answer-table tbody tr:hover {
+            background: #E8F4FD;
+        }
+
+        .answer-table tbody tr:nth-child(even) {
+            background: #F7F9FC;
+        }
+
+        .answer-table tbody tr:nth-child(even):hover {
+            background: #E8F4FD;
+        }
+    </style>
+</head>
+<body>
+    <h1>Enhanced Table Demo</h1>
+    <p>This demonstrates the new resizable table interface:</p>
+    
+    <div class="table-wrapper">
+        <div class="table-controls">
+            <div class="table-controls-left">
+                <div class="table-info">9 rows × 7 columns</div>
+            </div>
+            <div class="table-controls-right">
+                <div class="table-resize-controls">
+                    <button class="table-resize-btn" onclick="setTableSize('compact')">Compact</button>
+                    <button class="table-resize-btn active" onclick="setTableSize('comfortable')">Comfortable</button>
+                    <button class="table-resize-btn" onclick="setTableSize('spacious')">Spacious</button>
+                </div>
+            </div>
+        </div>
+        <div class="table-container">
+            <table class="answer-table comfortable">
+                <thead>
+                    <tr>
+                        <th>dept_name</th>
+                        <th>avg_salary</th>
+                        <th>num_employee</th>
+                        <th>male_salary</th>
+                        <th>female_salary</th>
+                        <th>avg_male_salary</th>
+                        <th>avg_female_salary</th>
+                    </tr>
+                </thead>
+                <tbody>
+                    <tr>
+                        <td>Sales</td>
+                        <td>$86,123.43</td>
+                        <td>130</td>
+                        <td>$6,181,954</td>
+                        <td>$5,014,084</td>
+                        <td>$84,684.37</td>
+                        <td>$87,966.44</td>
+                    </tr>
+                    <tr>
+                        <td>Marketing</td>
+                        <td>$81,874.03</td>
+                        <td>34</td>
+                        <td>$1,736,164</td>
+                        <td>$1,047,554</td>
+                        <td>$82,674.38</td>
+                        <td>$80,581.15</td>
+                    </tr>
+                    <tr>
+                        <td>Finance</td>
+                        <td>$76,399.98</td>
+                        <td>43</td>
+                        <td>$2,026,534</td>
+                        <td>$1,258,664</td>
+                        <td>$75,056.70</td>
+                        <td>$78,666.75</td>
+                    </tr>
+                    <tr>
+                        <td>Development</td>
+                        <td>$69,564.44</td>
+                        <td>203</td>
+                        <td>$8,159,084</td>
+                        <td>$5,962,504</td>
+                        <td>$69,735.73</td>
+                        <td>$69,331.42</td>
+                    </tr>
+                    <tr>
+                        <td>Research</td>
+                        <td>$69,147.39</td>
+                        <td>51</td>
+                        <td>$2,279,864</td>
+                        <td>$1,246,654</td>
+                        <td>$69,086.76</td>
+                        <td>$69,258.56</td>
+                    </tr>
+                    <tr>
+                        <td>Production</td>
+                        <td>$68,537.28</td>
+                        <td>186</td>
+                        <td>$7,929,244</td>
+                        <td>$4,818,694</td>
+                        <td>$68,355.55</td>
+                        <td>$68,838.43</td>
+                    </tr>
+                    <tr>
+                        <td>Quality Management</td>
+                        <td>$64,859.40</td>
+                        <td>48</td>
+                        <td>$2,112,644</td>
+                        <td>$1,000,604</td>
+                        <td>$64,019.61</td>
+                        <td>$66,706.93</td>
+                    </tr>
+                    <tr>
+                        <td>Customer Service</td>
+                        <td>$63,643.00</td>
+                        <td>61</td>
+                        <td>$1,730,834</td>
+                        <td>$2,151,394</td>
+                        <td>$61,815.43</td>
+                        <td>$65,193.67</td>
+                    </tr>
+                    <tr>
+                        <td>Human Resources</td>
+                        <td>$63,507.28</td>
+                        <td>46</td>
+                        <td>$2,151,834</td>
+                        <td>$769,499</td>
+                        <td>$63,289.29</td>
+                        <td>$64,124.92</td>
+                    </tr>
+                </tbody>
+            </table>
+        </div>
+    </div>
+
+    <script>
+        function setTableSize(size) {
+            const table = document.querySelector('.answer-table');
+            const buttons = document.querySelectorAll('.table-resize-btn');
+            
+            // Remove all size classes
+            table.classList.remove('compact', 'comfortable', 'spacious');
+            
+            // Add selected size class
+            table.classList.add(size);
+            
+            // Update active button
+            buttons.forEach(btn => btn.classList.remove('active'));
+            event.target.classList.add('active');
+        }
+    </script>
+</body>
+</html>