Deploy QueryLens V2 (code only)

.dockerignore

@@ -0,0 +1,89 @@
+# Exclude heavy folders from Docker image
+**/__pycache__
+**/.venv
+**/.git
+**/.ipynb_checkpoints
+**/hf_cache
+**/node_modules
+
+# Exclude local database data
+**/postgres-data
+**/mysql-data
+
+# Exclude large model weights and research artifacts
+Agent/Gemma3_1B_weights
+Agent/merged_gemma1b_text2Sql
+Agent/merged_gemma3_1b_text2Sql
+Agent/Qwen2.5_3b_weights
+Agent/Qwen2.5-3B-Instruct
+merged_qwen2_5_3b_text2sql
+merged_gemma3_1b_text2sql
+.venv-merge/
+
+# Exclude all Jupyter Notebooks EXCEPT the proof-of-work notebook
+**/*.ipynb
+!Fine_Tuning_Qwen_2_5_3B.ipynb
+**/.ipynb_checkpoints
+
+# Exclude redundant project clutter
+backups/
+debug_files/
+test_files/
+test_results/
+dump-mysql.bat
+dump-postgres.bat
+agentic_improvements.md
+invariants_analysis.md
+progress_summary.txt
+mcp_server.log
+merge.log
+response.json
+run-dev.ps1
+setup_ollama.bat
+
+
+# Exclude experimental/redundant Python scripts
+Agent/trial*.py
+Agent/test*.py
+Agent/old_*.py
+Agent/OLLAMA_GPU_NOTES.md
+**/debug_*.py
+
+# Exclude local uploads and generated files (keep empty folder)
+chat_store/*
+!chat_store/.gitkeep
+!uploads/.gitkeep
+
+# Exclude local configuration (Hugging Face uses Secrets instead)
+.env
+.env.local
+
+# Exclude local logs and IDE settings
+*.log
+.vscode/
+.idea/
+.DS_Store
+
+# Exclude Python build artifacts
+__pycache__/
+*.pyc
+*.pyo
+*.pyd
+.Python
+env/
+venv/
+.venv/
+pip-log.txt
+pip-delete-this-directory.txt
+.toast
+.mypy_cache/
+.pytest_cache/
+.hypothesis/
+
+# Exclude large local data
+uploads/*
+!uploads/.gitkeep
+chat_store/*
+!chat_store/docs/.gitkeep
+!chat_store/.gitkeep
+.idea/

.env.example

@@ -0,0 +1,29 @@
+# Core DB
+# DB_CONNECTION_URL=postgresql://user:pass@localhost:5432/dbname
+
+# ── Text-to-SQL LLM ──────────────────────────────────────────────
+# Recommended: HF Inference API (same as the Hugging Face Spaces deployment).
+# Set USE_HF_CLOUD=1 to force cloud-only. With USE_HF_CLOUD=0 the local Ollama
+# model is still preferred, but if it is not installed and HF_API_TOKEN is set,
+# the worker auto-falls back to HF Inference automatically.
+USE_HF_CLOUD=1
+HF_API_TOKEN=
+HF_LLM_MODEL=Qwen/Qwen2.5-Coder-7B-Instruct
+
+# Cloud VQA via Groq (free tier)
+GROQ_API_KEY=
+GROQ_VISION_MODEL=qwen/qwen3.8-27b
+
+# Local Ollama (optional — only when models are installed locally)
+# USE_HF_CLOUD=0
+# OLLAMA_BASE_URL=http://localhost:11434
+# OLLAMA_MODEL_NAME=qwen2.5-coder:1.5b
+# GRANITE_MODEL_PATH=qwen3.5:2b
+# OLLAMA_OCR_QA_MODEL=gemma3:1b-it-qat
+# GRANITE_MODEL=qwen3.5:2b
+
+# Workers/timeouts
+WORKER_PORT=8700
+MCP_PORT=8701
+OCR_QA_TIMEOUT=45
+INVARIANTS_BASE_TIMEOUT=180

.gitignore

@@ -0,0 +1,71 @@
+# Local Models and Weights
+Agent/*_weights/
+Agent/Qwen*/
+Agent/Gemma*/
+Agent/merged_*/
+merged_*/
+*.bin
+*.gguf
+*.onnx
+
+# Large Sample Database (Not needed in cloud) — EXCEPT the tiny bundled
+# demo SQLite file, which serves as the built-in demo_employee profile on
+# Hugging Face Spaces (no external DB service runs there).
+employee-sample-database/*
+!employee-sample-database/sqlite/
+employee-sample-database/sqlite/*
+!employee-sample-database/sqlite/dataset_small/
+employee-sample-database/sqlite/dataset_small/*
+!employee-sample-database/sqlite/dataset_small/employee.db
+
+# Data and History
+chat_store/
+Agent/chat_store_private.json
+
+# Docker and DB
+docker/data/
+
+# Python
+__pycache__/
+*.py[cod]
+*.egg-info/
+*.egg
+*.pyo
+*.pyd
+*.so
+
+# Virtual environments
+venv/
+.venv/
+.env
+.env.local
+!.env.example
+
+# Jupyter Notebooks and checkpoints
+*.ipynb
+.ipynb_checkpoints/
+
+# Experimental/Redundant Scripts
+Agent/trial*.py
+Agent/old_*.py
+Agent/requirements-merge.txt
+debug_files/
+test_results/
+
+# Data and Uploads (Keep the folders, ignore the contents)
+uploads/*
+!uploads/.gitkeep
+chat_store/docs/*
+!chat_store/docs/.gitkeep
+
+# OS-specific and IDEs
+.DS_Store
+Thumbs.db
+.vscode/
+.idea/
+
+# Logs
+*.log
+mcp_server.log
+merge.log
+response.json

Agent/chart_contract.py

@@ -0,0 +1,237 @@
+"""
+Chart contract — turns query result rows into a rendering-agnostic chart spec.
+
+The text2SQL pipeline produces plain result rows. This module detects rows that
+visualize well (category breakdowns, time series, comparisons) and produces a
+compact JSON spec embedded in the answer text as:
+
+    <<<CHART_JSON>>>{...spec...}<<<END_CHART_JSON>>>
+
+The worker strips the marker out of the user-visible answer and ships the spec
+separately. The web UI renders it with ECharts in the sepia palette. If no
+reliable vega-like shape is found, no marker is added and the UI falls back to
+tables/text — chart detection must never corrupt a correct answer.
+"""
+from __future__ import annotations
+
+import json
+import re
+from datetime import date, datetime
+from decimal import Decimal
+from typing import Any, Dict, List, Optional, Tuple
+
+CHART_OPEN = "<<<CHART_JSON>>>"
+CHART_CLOSE = "<<<END_CHART_JSON>>>"
+
+MAX_CHART_ROWS = 60          # cap categories drawn
+MAX_LABEL_LEN = 40           # truncate long labels
+PIE_MAX_ROWS = 8             # pies stay readable up to ~8 slices
+
+SERIES_NAME_HINTS = (
+    ("count", "count"), ("total", "total"), ("sum", "total"),
+    ("avg", "average"), ("average", "average"), ("min", "minimum"),
+    ("max", "maximum"), ("salary", "value"), ("amount", "value"),
+)
+
+DATE_RE = re.compile(
+    r"^\d{4}([-/.]\d{1,2}([-/.]\d{1,2})?)?$"          # 2024 / 2024-01 / 2024-01-01
+)
+
+
+# ---------------------------------------------------------------------------
+# Helpers
+# ---------------------------------------------------------------------------
+def _num(v: Any) -> Optional[float]:
+    if isinstance(v, bool):
+        return None
+    if isinstance(v, (int, float)) and v == v:  # NaN guard
+        return float(v)
+    if isinstance(v, Decimal):
+        try:
+            return float(v)
+        except Exception:
+            return None
+    if isinstance(v, str):
+        s = v.strip().replace(",", "")
+        if re.fullmatch(r"-?\d+(\.\d+)?", s):
+            try:
+                return float(s)
+            except ValueError:
+                return None
+    return None
+
+
+def _label(v: Any) -> str:
+    if v is None:
+        return "(none)"
+    if isinstance(v, (date, datetime)):
+        return str(v)[:MAX_LABEL_LEN]
+    s = str(v).strip()
+    if len(s) > MAX_LABEL_LEN:
+        s = s[: MAX_LABEL_LEN - 1] + "…"
+    return s
+
+
+def _is_dateish(v: Any) -> bool:
+    if isinstance(v, (date, datetime)):
+        return True
+    s = str(v).strip()
+    return bool(s) and bool(DATE_RE.match(s))
+
+
+def _nice_series_name(col: str) -> str:
+    c = (col or "").strip()
+    cl = c.lower()
+    for key, nice in SERIES_NAME_HINTS:
+        if key in cl:
+            base = c.split("(")[0].strip()  # strip "AVG(" noise in aliases
+            return base or nice
+    return c
+
+
+# ---------------------------------------------------------------------------
+# Detection
+# ---------------------------------------------------------------------------
+def detect_chart_spec(results: List[Dict[str, Any]], question: str = "") -> Optional[Dict[str, Any]]:
+    """Return a chart spec dict when `results` visualize well, else None.
+
+    Spec shape (ECharts-friendly, library-agnostic):
+    {
+      "type": "pie" | "bar" | "line",
+      "title": str,             # derived from the question
+      "xlabel": str, "ylabel": str,
+      "x": [labels],            # categories or time points
+      "series": [{"name": str, "values": [numbers]}],
+      "rows_in_result": int
+    }
+    """
+    if not results or not isinstance(results, list):
+        return None
+    if not isinstance(results[0], dict) or results[0].get("error"):
+        return None
+
+    rows = [r for r in results if isinstance(r, dict)]
+    if len(rows) > MAX_CHART_ROWS:
+        return None
+
+    cols = list(rows[0].keys())
+    if not cols:
+        return None
+
+    label_col: Optional[str] = None
+    time_col: Optional[str] = None
+    numeric_cols: List[str] = []
+
+    for c in cols:
+        vals = [r.get(c) for r in rows]
+        non_none = [v for v in vals if v is not None]
+        if not non_none:
+            continue
+        # Date-like strings take precedence over numeric-string coercion
+        # (a "2020"/"2024-01" column is a time axis, not a measurement).
+        all_dateish = all(_is_dateish(v) for v in non_none)
+        if all_dateish and time_col is None and label_col is None:
+            time_col = c
+            continue
+        nums = [_num(v) for v in vals]
+        n_numeric = sum(1 for n in nums if n is not None)
+        if n_numeric == len(non_none):
+            numeric_cols.append(c)
+        elif label_col is None:
+            uniq = {str(v) for v in non_none}
+            if 0 < len(uniq) <= MAX_CHART_ROWS:  # degenerate constant columns still ok
+                label_col = c
+
+    if time_col:
+        label_col = time_col
+
+    if label_col is None or not numeric_cols:
+        return None
+
+    labels = [_label(r.get(label_col)) for r in rows]
+
+    series: List[Dict[str, Any]] = []
+    for c in numeric_cols[:3]:  # at most 3 series keeps charts readable
+        values: List[Optional[float]] = []
+        ok = True
+        for r in rows:
+            n = _num(r.get(c))
+            if n is None:
+                ok = False
+                break
+            values.append(n)
+        if ok and any(v != 0 for v in values):
+            values = [round(v, 6) for v in values]
+            series.append({"name": _nice_series_name(c), "values": values})
+
+    if not series:
+        return None
+
+    # Chart type selection --------------------------------------------------
+    ql = (question or "").lower()
+    proportion_intent = any(
+        k in ql for k in ("distribution", "share", "percentage", "percent", "proportion", "%", "breakdown", "split")
+    )
+    trend_intent = time_col is not None or any(
+        k in ql for k in ("trend", "over time", "per year", "by year", "per month", "by month", "growth")
+    )
+
+    single_series = len(series) == 1
+    if single_series and len(labels) <= PIE_MAX_ROWS and (proportion_intent or trend_intent is False):
+        chart_type = "pie"
+    elif trend_intent:
+        chart_type = "line"
+    elif single_series and len(labels) <= PIE_MAX_ROWS and len(rows) <= PIE_MAX_ROWS:
+        chart_type = "pie"
+    else:
+        chart_type = "bar"
+
+    title = (question or "").strip().rstrip("?").strip()
+    if len(title) > 80:
+        title = title[:79] + "…"
+
+    return {
+        "type": chart_type,
+        "title": title or "Query result",
+        "xlabel": _nice_series_name(label_col),
+        "ylabel": series[0]["name"] if single_series else "value",
+        "x": labels,
+        "series": series,
+        "rows_in_result": len(rows),
+    }
+
+
+# ---------------------------------------------------------------------------
+# Embedding / extraction
+# ---------------------------------------------------------------------------
+def embed_chart_marker(answer_text: str, spec: Optional[Dict[str, Any]]) -> str:
+    """Append the chart marker to an answer. Safe to call with spec=None."""
+    if not spec:
+        return answer_text
+    try:
+        payload = json.dumps(spec, ensure_ascii=False, separators=(",", ":"), default=str)
+    except (TypeError, ValueError):
+        return answer_text
+    return f"{answer_text}\n{CHART_OPEN}{payload}{CHART_CLOSE}"
+
+
+def extract_chart_marker(text: Optional[str]) -> Tuple[Optional[str], Optional[Dict[str, Any]]]:
+    """Split an answer into (clean_text, chart_spec|None).
+
+    Never throws; on malformed payloads the marker text is left in the answer
+    if extraction fails (chart simply won't render).
+    """
+    if not text or CHART_OPEN not in text:
+        return text, None
+    head, rest = text.split(CHART_OPEN, 1)
+    if CHART_CLOSE not in rest:
+        return text, None
+    payload, tail = rest.split(CHART_CLOSE, 1)
+    clean = (head.rstrip() + ("\n" + tail.strip() if tail.strip() else "")).strip()
+    try:
+        spec = json.loads(payload)
+        if isinstance(spec, dict) and spec.get("type") in ("pie", "bar", "line") and spec.get("series"):
+            return clean, spec
+    except (ValueError, TypeError):
+        pass
+    return text, None

Agent/invariants.py

@@ -0,0 +1,499 @@
+"""Invariant classification and enforcement for text2sql without hardcoded SQL templates.
+
+All regeneration remains purely LLM-driven. We only express semantic/structural
+constraints ("invariants") and ask the model to satisfy missing ones.
+
+Categories:
+  MANAGER_LIST          -> list current department managers
+  DEPARTMENT_LIST       -> list department names
+  AVG_SALARY_BY_DEPT    -> average current salary per department
+  TITLE_COUNT           -> count employees with a given title (e.g. Senior Engineer)
+  GENERIC               -> anything else (no special invariants)
+
+Each invariant is represented as a dict:
+  {
+    'name': short identifier,
+    'description': human-readable guidance,
+    'check': callable(sql:str)->bool
+  }
+
+We DO NOT inject SQL fragments. We only:
+  - detect which invariants fail
+  - compile a focused regeneration prompt listing just the missing invariants
+  - request a corrected SQL (single statement ending with semicolon + LIMIT 100)
+
+This preserves the "no hardcoded fallback query" policy.
+"""
+from __future__ import annotations
+import os
+import re
+from typing import List, Dict, Callable, Optional
+
+CATEGORY_MANAGER_LIST = "MANAGER_LIST"
+CATEGORY_DEPT_LIST = "DEPARTMENT_LIST"
+CATEGORY_AVG_SALARY = "AVG_SALARY_BY_DEPT"
+CATEGORY_TITLE_COUNT = "TITLE_COUNT"
+CATEGORY_EMP_COUNT_BY_DEPT = "EMP_COUNT_BY_DEPT"
+CATEGORY_GENERIC = "GENERIC"
+
+# Newly added categories (extended taxonomy)
+CATEGORY_SIMPLE_COUNT_EMP = "SIMPLE_COUNT_EMP"
+CATEGORY_SIMPLE_COUNT_EMP_CURRENT = "SIMPLE_COUNT_EMP_CURRENT"
+CATEGORY_SIMPLE_COUNT_DEPT = "SIMPLE_COUNT_DEPT"
+CATEGORY_SALARY_EXTREMES_GLOBAL = "SALARY_EXTREMES_GLOBAL"
+CATEGORY_SALARY_RANGE_BY_DEPT = "SALARY_RANGE_BY_DEPT"
+CATEGORY_TOP_N_SALARIES = "TOP_N_SALARIES"
+CATEGORY_DEPT_MANAGER_LIST = "DEPT_MANAGER_LIST"  # dept + manager names
+CATEGORY_MANAGER_TENURE = "MANAGER_TENURE"
+CATEGORY_MANAGER_CHANGE_FREQUENCY = "MANAGER_CHANGE_FREQUENCY"
+CATEGORY_TITLE_DISTINCT_COUNT = "TITLE_DISTINCT_COUNT"
+CATEGORY_TITLE_SPECIFIC_COUNT = "TITLE_SPECIFIC_COUNT"
+CATEGORY_TITLE_DISTRIBUTION_BY_DEPT = "TITLE_DISTRIBUTION_BY_DEPT"
+CATEGORY_PROMOTION_PATH_ENGINEER_TO_SENIOR = "PROMOTION_PATH_ENGINEER_TO_SENIOR"
+CATEGORY_EMP_TENURE_CURRENT_DEPT = "EMP_TENURE_CURRENT_DEPT"
+CATEGORY_HIRING_TREND_BY_YEAR = "HIRING_TREND_BY_YEAR"
+CATEGORY_SALARY_TREND_BY_YEAR = "SALARY_TREND_BY_YEAR"
+CATEGORY_EMP_DEPT_CHANGES = "EMP_DEPT_CHANGES"
+CATEGORY_GENDER_DIST_BY_DEPT = "GENDER_DIST_BY_DEPT"
+CATEGORY_GENDER_PAY_GAP_BY_DEPT = "GENDER_PAY_GAP_BY_DEPT"
+CATEGORY_FEMALE_MANAGERS_COUNT = "FEMALE_MANAGERS_COUNT"
+CATEGORY_AVG_SALARY_BY_TITLE_AND_DEPT = "AVG_SALARY_BY_TITLE_AND_DEPT"
+CATEGORY_GENDER_MANAGEMENT_COUNT = "GENDER_MANAGEMENT_COUNT"
+CATEGORY_SENIOR_STAFF_SUPERLATIVE = "SENIOR_STAFF_SUPERLATIVE"
+
+Category = str
+
+# ---------------- Classification -----------------
+
+def classify_question(question: str) -> Category:
+    q = question.lower()
+    # Specific order matters: more specialized patterns first
+    if any(k in q for k in ["top", "highest-paid", "highest paid"]) and "salary" in q:
+        return CATEGORY_TOP_N_SALARIES
+    if any(kw in q for kw in ["total number of employees", "total employees"]) or ("count" in q and "employee" in q and "department" not in q and "current" not in q):
+        return CATEGORY_SIMPLE_COUNT_EMP
+    if ("how many" in q or "count" in q) and "currently" in q and "employee" in q:
+        return CATEGORY_SIMPLE_COUNT_EMP_CURRENT
+    if ("how many" in q or "number of" in q or "count" in q) and "departments" in q and "manager" not in q:
+        return CATEGORY_SIMPLE_COUNT_DEPT
+    if any(w in q for w in ["highest", "lowest"]) and "salary" in q and "department" not in q:
+        return CATEGORY_SALARY_EXTREMES_GLOBAL
+    if "salary range" in q and "department" in q:
+        return CATEGORY_SALARY_RANGE_BY_DEPT
+    if "most employees" in q and "department" in q:
+        return CATEGORY_EMP_COUNT_BY_DEPT
+    if "highest average salary" in q and "department" in q:
+        return CATEGORY_AVG_SALARY
+    if any(k in q for k in ["manager", "managers"]) and "department" in q:
+        if "tenure" in q or "long" in q or "position" in q:
+            return CATEGORY_MANAGER_TENURE
+        if "changed" in q or "changes" in q or "frequently" in q:
+            return CATEGORY_MANAGER_CHANGE_FREQUENCY
+        if "list" in q and "manager" in q and "department" in q:
+            return CATEGORY_DEPT_MANAGER_LIST
+        return CATEGORY_MANAGER_LIST
+    if ("how many" in q or "count" in q) and "job title" in q and ("exist" in q or "different" in q):
+        return CATEGORY_TITLE_DISTINCT_COUNT
+    if ("how many" in q or "count" in q) and any(t in q for t in ["senior engineer", "senior engineers"]):
+        return CATEGORY_TITLE_SPECIFIC_COUNT
+    if "distribution" in q and "title" in q and "department" in q:
+        return CATEGORY_TITLE_DISTRIBUTION_BY_DEPT
+    if "promoted" in q and "engineer" in q and "senior engineer" in q:
+        return CATEGORY_PROMOTION_PATH_ENGINEER_TO_SENIOR
+    if ("how long" in q or "tenure" in q) and "current" in q and "department" in q and "employee" in q:
+        return CATEGORY_EMP_TENURE_CURRENT_DEPT
+    if ("hiring trend" in q) or ("hire" in q and "year" in q):
+        return CATEGORY_HIRING_TREND_BY_YEAR
+    if ("salaries" in q or "salary" in q) and "changed" in q and ("over time" in q or "year" in q):
+        return CATEGORY_SALARY_TREND_BY_YEAR
+    if "changed departments" in q or ("employees" in q and "changed" in q and "department" in q):
+        return CATEGORY_EMP_DEPT_CHANGES
+    if "gender distribution" in q and "department" in q:
+        return CATEGORY_GENDER_DIST_BY_DEPT
+    if "gender pay gap" in q and "department" in q:
+        return CATEGORY_GENDER_PAY_GAP_BY_DEPT
+    if ("female" in q and "manager" in q and ("how many" in q or "count" in q)):
+        return CATEGORY_FEMALE_MANAGERS_COUNT
+    if "average salary" in q and "title" in q and "department" in q:
+        return CATEGORY_AVG_SALARY_BY_TITLE_AND_DEPT
+    if ("gender" in q and "management" in q) or ("each gender" in q and "management" in q):
+        return CATEGORY_GENDER_MANAGEMENT_COUNT
+    if ("most" in q and "senior" in q and "staff" in q) or ("senior staff" in q and "most" in q):
+        return CATEGORY_SENIOR_STAFF_SUPERLATIVE
+    if any(k in q for k in ["list departments", "department names", "department name", "departments list"]):
+        return CATEGORY_DEPT_LIST
+    if "average" in q and "salary" in q and "department" in q:
+        return CATEGORY_AVG_SALARY
+    if ("count" in q or "how many" in q or "number of" in q) and any(t in q for t in ["engineer", "engineers", "title"]):
+        return CATEGORY_TITLE_COUNT
+    if ("count" in q or "how many" in q or "number of" in q or any(s in q for s in ["most", "largest", "highest", "fewest", "least"])) and "department" in q and ("employee" in q or "employees" in q):
+        return CATEGORY_EMP_COUNT_BY_DEPT
+    return CATEGORY_GENERIC
+
+# --------------- Invariant Builders ---------------
+
+def _regex(pattern: str) -> Callable[[str], bool]:
+    rx = re.compile(pattern, re.IGNORECASE)
+    return lambda sql: bool(rx.search(sql))
+
+def _contains_all(*subs: str) -> Callable[[str], bool]:
+    def check(sql: str) -> bool:
+        l = sql.lower()
+        return all(s.lower() in l for s in subs)
+    return check
+
+def build_invariants(category: Category, question: str) -> List[Dict[str, any]]:
+    inv: List[Dict[str, any]] = []
+    if category == CATEGORY_MANAGER_LIST:
+        inv += [
+            {"name": "has_department_join", "description": "Join department (alias d) with dept_manager (alias dm)", "check": lambda s: re.search(r"join\s+dept_manager", s, re.IGNORECASE) and re.search(r"from\s+department", s, re.IGNORECASE)},
+            {"name": "has_employee_join", "description": "Join employee (alias e) for manager names", "check": _regex(r"join\s+employee")},
+            {"name": "selects_names", "description": "Select employee first_name and last_name", "check": lambda s: all(x in s.lower() for x in ["first_name", "last_name"])},
+            {"name": "filters_current", "description": "Filter current managers dm.to_date='9999-01-01'", "check": _regex(r"dm\.to_date\s*=\s*'9999-01-01'")},
+            {"name": "limit_100", "description": "Ends with LIMIT 100 semicolon", "check": _regex(r"limit\s+100;?\s*$")},
+        ]
+    elif category == CATEGORY_DEPT_LIST:
+        inv += [
+            {"name": "from_department", "description": "Query FROM department table", "check": _regex(r"from\s+department")},
+            {"name": "select_dept_name", "description": "Select dept_name column", "check": _regex(r"dept_name")},
+            {"name": "limit_100", "description": "Ends with LIMIT 100", "check": _regex(r"limit\s+100;?\s*$")},
+        ]
+    elif category == CATEGORY_AVG_SALARY:
+        inv += [
+            {"name": "joins_salary", "description": "Join salary s (FROM salary s)", "check": _regex(r"from\s+salary\s+s")},
+            {"name": "joins_dept_emp", "description": "Join dept_emp de", "check": _regex(r"join\s+dept_emp")},
+            {"name": "joins_department", "description": "Join department d", "check": _regex(r"join\s+department")},
+            {"name": "avg_salary", "description": "Uses AVG(s.salary)", "check": _regex(r"avg\s*\(\s*s\.salary\s*\)")},
+            {"name": "current_filters", "description": "Filters current rows s.to_date='9999-01-01' and de.to_date='9999-01-01'", "check": lambda s: "s.to_date='9999-01-01'" in s.lower() and "de.to_date='9999-01-01'" in s.lower()},
+            {"name": "group_by_dept", "description": "GROUP BY department name", "check": _regex(r"group\s+by\s+.*dept_name")},
+            {"name": "limit_100", "description": "Ends with LIMIT 100", "check": _regex(r"limit\s+100;?\s*$")},
+        ]
+    elif category == CATEGORY_TITLE_COUNT:
+        # Attempt to infer canonical title (not used in invariants; regeneration prompt may hint)
+        inv += [
+            {"name": "from_title", "description": "FROM title table (alias t)", "check": _regex(r"from\s+title")},
+            {"name": "alias_t", "description": "Alias 't' used for title table", "check": _regex(r"from\s+title\s+(?:as\s+)?t")},
+            {"name": "current_rows", "description": "Filter current rows t.to_date='9999-01-01'", "check": _regex(r"t\.to_date\s*=\s*'9999-01-01'")},
+            {"name": "count_distinct", "description": "COUNT(DISTINCT t.emp_no)", "check": _regex(r"count\s*\(\s*distinct\s+t\.emp_no\s*\)")},
+            {"name": "has_title_filter", "description": "Filter on t.title = '<Some Title>'", "check": _regex(r"t\.title\s*=\s*'[^']+'")} ,
+            {"name": "limit_100", "description": "Ends with LIMIT 100", "check": _regex(r"limit\s+100;?\s*$")},
+        ]
+    elif category == CATEGORY_EMP_COUNT_BY_DEPT:
+        inv += [
+            {"name": "has_dept_emp_join", "description": "Join dept_emp de with department d", "check": lambda s: re.search(r"join\s+dept_emp", s, re.IGNORECASE) and re.search(r"join\s+department|from\s+department", s, re.IGNORECASE)},
+            {"name": "select_dept_name", "description": "Select d.dept_name", "check": _regex(r"d\.dept_name|dept_name")},
+            {"name": "select_count", "description": "Include COUNT(*) or COUNT(de.emp_no)", "check": _regex(r"count\s*\(")},
+            {"name": "group_by", "description": "GROUP BY d.dept_name present", "check": _regex(r"group\s+by\s+.*dept_name")},
+            {"name": "limit_100", "description": "Ends with LIMIT 100", "check": _regex(r"limit\s+100;?\s*$")},
+            {"name": "count_alias", "description": "Provide an explicit alias e.g. COUNT(*) AS employee_count", "check": _regex(r"count\s*\(.*\)\s+as\s+employee_count")},
+        ]
+        # If the question is superlative-oriented, encourage ordering
+        ql = question.lower()
+        if any(w in ql for w in ["most", "largest", "highest", "fewest", "least"]):
+            inv.append({"name": "order_by_count", "description": "ORDER BY employee_count DESC (or ASC for fewest/least)", "check": lambda s: ("fewest" in ql or "least" in ql and re.search(r"order\s+by\s+employee_count\s+asc", s, re.IGNORECASE)) or re.search(r"order\s+by\s+employee_count\s+desc", s, re.IGNORECASE)})
+    elif category == CATEGORY_SIMPLE_COUNT_EMP:
+        inv += [
+            {"name": "from_employee", "description": "FROM employee table", "check": _regex(r"from\s+employee")},
+            {"name": "count_all", "description": "COUNT(*) with alias total_employees", "check": _regex(r"count\s*\(\s*\*\s*\)\s+as\s+total_employees")},
+            {"name": "limit_100", "description": "Ends with LIMIT 100", "check": _regex(r"limit\s+100;?\s*$")},
+        ]
+    elif category == CATEGORY_SIMPLE_COUNT_EMP_CURRENT:
+        inv += [
+            {"name": "join_dept_emp", "description": "Join dept_emp de", "check": _regex(r"join\s+dept_emp")},
+            {"name": "current_filter", "description": "de.to_date='9999-01-01' filter", "check": _regex(r"de\.to_date\s*=\s*'9999-01-01'")},
+            {"name": "count_distinct_emp", "description": "COUNT(DISTINCT de.emp_no) alias current_employee_count", "check": _regex(r"count\s*\(\s*distinct\s+de\.emp_no\s*\)\s+as\s+current_employee_count")},
+            {"name": "limit_100", "description": "Ends with LIMIT 100", "check": _regex(r"limit\s+100;?\s*$")},
+        ]
+    elif category == CATEGORY_SIMPLE_COUNT_DEPT:
+        inv += [
+            {"name": "from_department", "description": "FROM department", "check": _regex(r"from\s+department")},
+            {"name": "count_depts", "description": "COUNT(*) AS department_count", "check": _regex(r"count\s*\(\s*\*\s*\)\s+as\s+department_count")},
+            {"name": "limit_100", "description": "Ends with LIMIT 100", "check": _regex(r"limit\s+100;?\s*$")},
+        ]
+    elif category == CATEGORY_SALARY_EXTREMES_GLOBAL:
+        # Extremely strict invariants for highest/lowest salary by department
+        inv += [
+            {"name": "from_salary_s", "description": "FROM salary s", "check": _regex(r"from\s+salary\s+s")},
+            {"name": "join_dept_emp_de", "description": "JOIN dept_emp de ON s.emp_no = de.emp_no", "check": _regex(r"join\s+dept_emp\s+de\s+on\s+s\.emp_no\s*=\s*de\.emp_no")},
+            {"name": "join_department_d", "description": "JOIN department d ON de.dept_no = d.dept_no", "check": _regex(r"join\s+department\s+d\s+on\s+de\.dept_no\s*=\s*d\.dept_no")},
+            {"name": "select_dept_name", "description": "SELECT d.dept_name", "check": _regex(r"select\s+d\.dept_name")},
+            {"name": "select_max_min_salary", "description": "SELECT MAX(s.salary) AS highest_salary, MIN(s.salary) AS lowest_salary", "check": lambda s: re.search(r"max\s*\(\s*s\.salary\s*\)\s+as\s+highest_salary", s, re.IGNORECASE) and re.search(r"min\s*\(\s*s\.salary\s*\)\s+as\s+lowest_salary", s, re.IGNORECASE)},
+            {"name": "group_by_dept_name", "description": "GROUP BY d.dept_name", "check": _regex(r"group\s+by\s+d\.dept_name")},
+            {"name": "limit_100", "description": "Ends with LIMIT 100", "check": _regex(r"limit\s+100;?\s*$")},
+            {"name": "no_unaliased_salary", "description": "Do not use MAX(salary) or MIN(salary) without s. alias", "check": lambda s: not re.search(r"max\s*\(\s*salary\s*\)", s, re.IGNORECASE) and not re.search(r"min\s*\(\s*salary\s*\)", s, re.IGNORECASE)},
+            {"name": "no_unaliased_dept_name", "description": "Do not use dept_name without d. alias", "check": lambda s: not re.search(r"[^a-zA-Z0-9_]dept_name", s) or re.search(r"d\.dept_name", s)},
+        ]
+    elif category == CATEGORY_SALARY_RANGE_BY_DEPT:
+        # Stricter invariants for salary range by department
+        inv += [
+            {"name": "joins_salary", "description": "FROM salary s", "check": _regex(r"from\s+salary\s+s")},
+            {"name": "joins_dept_emp", "description": "Join dept_emp de", "check": _regex(r"join\s+dept_emp")},
+            {"name": "joins_department", "description": "Join department d", "check": _regex(r"join\s+department")},
+            {"name": "select_dept_name", "description": "Select d.dept_name", "check": _regex(r"d\.dept_name")},
+            {"name": "min_max", "description": "MIN(s.salary) & MAX(s.salary)", "check": lambda s: re.search(r"min\s*\(\s*s\.salary", s, re.IGNORECASE) and re.search(r"max\s*\(\s*s\.salary", s, re.IGNORECASE)},
+            {"name": "dept_filter", "description": "Filter on d.dept_name=...", "check": _regex(r"d\.dept_name\s*=\s*'")},
+            {"name": "group_by_dept", "description": "GROUP BY d.dept_name", "check": _regex(r"group\s+by\s+.*d\.dept_name")},
+            {"name": "limit_100", "description": "Ends with LIMIT 100", "check": _regex(r"limit\s+100;?\s*$")},
+        ]
+    elif category == CATEGORY_TOP_N_SALARIES:
+        inv += [
+            {"name": "joins_salary", "description": "FROM salary s", "check": _regex(r"from\s+salary\s+s")},
+            {"name": "current_filter", "description": "s.to_date='9999-01-01'", "check": _regex(r"s\.to_date\s*=\s*'9999-01-01'")},
+            {"name": "order_desc", "description": "ORDER BY s.salary DESC", "check": _regex(r"order\s+by\s+s\.salary\s+desc")},
+            {"name": "limit_present", "description": "Has a LIMIT clause", "check": _regex(r"limit\s+\d+")},
+        ]
+    elif category == CATEGORY_DEPT_MANAGER_LIST:
+        inv += [
+            {"name": "joins_dept_manager", "description": "Join dept_manager dm", "check": _regex(r"join\s+dept_manager")},
+            {"name": "joins_employee", "description": "Join employee e", "check": _regex(r"join\s+employee")},
+            {"name": "select_names", "description": "Select manager first_name / last_name", "check": lambda s: "first_name" in s.lower() and "last_name" in s.lower()},
+            {"name": "dept_name", "description": "Select dept_name", "check": _regex(r"dept_name")},
+            {"name": "current_filter", "description": "dm.to_date='9999-01-01'", "check": _regex(r"dm\.to_date\s*=\s*'9999-01-01'")},
+            {"name": "limit_100", "description": "Ends with LIMIT 100", "check": _regex(r"limit\s+100;?\s*$")},
+        ]
+    elif category == CATEGORY_MANAGER_TENURE:
+        inv += [
+            {"name": "joins_dept_manager", "description": "Join dept_manager dm", "check": _regex(r"join\s+dept_manager")},
+            {"name": "current_filter", "description": "dm.to_date='9999-01-01'", "check": _regex(r"dm\.to_date\s*=\s*'9999-01-01'")},
+            {"name": "tenure_expr", "description": "Contains dm.from_date for tenure calc", "check": _regex(r"dm\.from_date")},
+            {"name": "limit_100", "description": "Ends with LIMIT 100", "check": _regex(r"limit\s+100;?\s*$")},
+        ]
+    elif category == CATEGORY_MANAGER_CHANGE_FREQUENCY:
+        inv += [
+            {"name": "from_dept_manager", "description": "FROM dept_manager", "check": _regex(r"from\s+dept_manager")},
+            {"name": "group_by_dept", "description": "GROUP BY dept_no or dept", "check": _regex(r"group\s+by\s+.*dept")},
+            {"name": "count_changes", "description": "COUNT(*) or COUNT(DISTINCT emp_no)", "check": _regex(r"count\s*\(")},
+            {"name": "order_desc", "description": "ORDER BY count desc", "check": _regex(r"order\s+by\s+.*count.*desc")},
+            {"name": "limit_100", "description": "Ends with LIMIT 100", "check": _regex(r"limit\s+100;?\s*$")},
+        ]
+    elif category == CATEGORY_TITLE_DISTINCT_COUNT:
+        inv += [
+            {"name": "from_title", "description": "FROM title t", "check": _regex(r"from\s+title")},
+            {"name": "count_distinct_titles", "description": "COUNT(DISTINCT t.title) alias title_count", "check": _regex(r"count\s*\(\s*distinct\s+t\.title\s*\)\s+as\s+title_count")},
+            {"name": "limit_100", "description": "Ends with LIMIT 100", "check": _regex(r"limit\s+100;?\s*$")},
+        ]
+    elif category == CATEGORY_TITLE_SPECIFIC_COUNT:
+        inv += [
+            {"name": "from_title", "description": "FROM title t", "check": _regex(r"from\s+title")},
+            {"name": "title_filter", "description": "Filter t.title='Senior Engineer'", "check": _regex(r"t\.title\s*=\s*'senior engineer'")},
+            {"name": "current_filter", "description": "t.to_date='9999-01-01'", "check": _regex(r"t\.to_date\s*=\s*'9999-01-01'")},
+            {"name": "count_distinct_emp", "description": "COUNT(DISTINCT t.emp_no)", "check": _regex(r"count\s*\(\s*distinct\s+t\.emp_no")},
+            {"name": "limit_100", "description": "Ends with LIMIT 100", "check": _regex(r"limit\s+100;?\s*$")},
+        ]
+    elif category == CATEGORY_TITLE_DISTRIBUTION_BY_DEPT:
+        inv += [
+            {"name": "joins_title", "description": "Join title t", "check": _regex(r"join\s+title")},
+            {"name": "joins_dept_emp", "description": "Join dept_emp de", "check": _regex(r"join\s+dept_emp")},
+            {"name": "joins_department", "description": "Join department d", "check": _regex(r"join\s+department")},
+            {"name": "current_filters", "description": "t.to_date & de.to_date current filters", "check": lambda s: "t.to_date='9999-01-01'" in s.lower() and "de.to_date='9999-01-01'" in s.lower()},
+            {"name": "group_by", "description": "GROUP BY dept and title", "check": _regex(r"group\s+by\s+.*(dept|title).*(title|dept)")},
+            {"name": "limit_100", "description": "Ends with LIMIT 100", "check": _regex(r"limit\s+100;?\s*$")},
+        ]
+    elif category == CATEGORY_PROMOTION_PATH_ENGINEER_TO_SENIOR:
+        inv += [
+            {"name": "dual_title_refs", "description": "References title table twice (t1, t2)", "check": _regex(r"title\s+t1.*title\s+t2|title\s+t2.*title\s+t1")},
+            {"name": "promotion_condition", "description": "t1.title='Engineer' AND t2.title='Senior Engineer'", "check": _regex(r"t1\.title\s*=\s*'engineer'.*t2\.title\s*=\s*'senior engineer'|t2\.title\s*=\s*'senior engineer'.*t1\.title\s*=\s*'engineer'")},
+            {"name": "temporal_order", "description": "t1.from_date < t2.from_date", "check": _regex(r"t1\.from_date\s*<\s*t2\.from_date")},
+            {"name": "distinct_emp", "description": "DISTINCT emp_no selected", "check": _regex(r"select\s+distinct\s+t1\.emp_no|select\s+distinct\s+t2\.emp_no")},
+            {"name": "limit_100", "description": "Ends with LIMIT 100", "check": _regex(r"limit\s+100;?\s*$")},
+        ]
+    elif category == CATEGORY_EMP_TENURE_CURRENT_DEPT:
+        inv += [
+            {"name": "joins_dept_emp", "description": "FROM dept_emp de", "check": _regex(r"from\s+dept_emp")},
+            {"name": "current_filter", "description": "de.to_date='9999-01-01'", "check": _regex(r"de\.to_date\s*=\s*'9999-01-01'")},
+            {"name": "tenure_expr", "description": "Uses de.from_date", "check": _regex(r"de\.from_date")},
+            {"name": "limit_100", "description": "Ends with LIMIT 100", "check": _regex(r"limit\s+100;?\s*$")},
+        ]
+    elif category == CATEGORY_HIRING_TREND_BY_YEAR:
+        inv += [
+            {"name": "from_employee", "description": "FROM employee e", "check": _regex(r"from\s+employee")},
+            {"name": "year_extract", "description": "EXTRACT(YEAR FROM hire_date)", "check": _regex(r"extract\s*\(\s*year\s+from\s+.*hire_date")},
+            {"name": "group_by_year", "description": "GROUP BY year", "check": _regex(r"group\s+by\s+.*year")},
+            {"name": "limit_100", "description": "Ends with LIMIT 100", "check": _regex(r"limit\s+100;?\s*$")},
+        ]
+    elif category == CATEGORY_SALARY_TREND_BY_YEAR:
+        inv += [
+            {"name": "from_salary", "description": "FROM salary s", "check": _regex(r"from\s+salary")},
+            {"name": "year_extract", "description": "EXTRACT(YEAR FROM s.from_date)", "check": _regex(r"extract\s*\(\s*year\s+from\s+s\.from_date")},
+            {"name": "avg_salary", "description": "AVG(s.salary)", "check": _regex(r"avg\s*\(\s*s\.salary")},
+            {"name": "group_by_year", "description": "GROUP BY year", "check": _regex(r"group\s+by\s+.*year")},
+            {"name": "limit_100", "description": "Ends with LIMIT 100", "check": _regex(r"limit\s+100;?\s*$")},
+        ]
+    elif category == CATEGORY_EMP_DEPT_CHANGES:
+        inv += [
+            {"name": "from_dept_emp", "description": "FROM dept_emp de", "check": _regex(r"from\s+dept_emp")},
+            {"name": "group_by_emp", "description": "GROUP BY de.emp_no", "check": _regex(r"group\s+by\s+.*emp_no")},
+            {"name": "having_multiple", "description": "HAVING COUNT(DISTINCT dept_no) > 1", "check": _regex(r"having\s+count\s*\(\s*distinct\s+dept_no\s*\)\s*>\s*1")},
+            {"name": "limit_100", "description": "Ends with LIMIT 100", "check": _regex(r"limit\s+100;?\s*$")},
+        ]
+    elif category == CATEGORY_GENDER_DIST_BY_DEPT:
+        inv += [
+            {"name": "joins_employee", "description": "Join employee e", "check": _regex(r"join\s+employee")},
+            {"name": "joins_dept_emp", "description": "Join dept_emp de", "check": _regex(r"join\s+dept_emp")},
+            {"name": "joins_department", "description": "Join department d", "check": _regex(r"join\s+department")},
+            {"name": "current_filter", "description": "de.to_date='9999-01-01'", "check": _regex(r"de\.to_date\s*=\s*'9999-01-01'")},
+            {"name": "group_by_dept_gender", "description": "GROUP BY dept & gender", "check": _regex(r"group\s+by\s+.*(dept|gender).*(gender|dept)")},
+            {"name": "limit_100", "description": "Ends with LIMIT 100", "check": _regex(r"limit\s+100;?\s*$")},
+        ]
+    elif category == CATEGORY_GENDER_PAY_GAP_BY_DEPT:
+        inv += [
+            {"name": "joins_salary", "description": "Join salary s", "check": _regex(r"join\s+salary")},
+            {"name": "joins_employee", "description": "Join employee e", "check": _regex(r"join\s+employee")},
+            {"name": "joins_dept_emp", "description": "Join dept_emp de", "check": _regex(r"join\s+dept_emp")},
+            {"name": "joins_department", "description": "Join department d", "check": _regex(r"join\s+department")},
+            {"name": "current_filters", "description": "Current rows s.to_date & de.to_date", "check": lambda s: "s.to_date='9999-01-01'" in s.lower() and "de.to_date='9999-01-01'" in s.lower()},
+            {"name": "avg_salary", "description": "AVG(s.salary)", "check": _regex(r"avg\s*\(\s*s\.salary")},
+            {"name": "group_by_dept_gender", "description": "GROUP BY dept & gender", "check": _regex(r"group\s+by\s+.*(dept|gender).*(gender|dept)")},
+            {"name": "limit_100", "description": "Ends with LIMIT 100", "check": _regex(r"limit\s+100;?\s*$")},
+        ]
+    elif category == CATEGORY_FEMALE_MANAGERS_COUNT:
+        inv += [
+            {"name": "joins_dept_manager", "description": "Join dept_manager dm", "check": _regex(r"join\s+dept_manager")},
+            {"name": "joins_employee", "description": "Join employee e", "check": _regex(r"join\s+employee")},
+            {"name": "female_filter", "description": "e.gender='F'", "check": _regex(r"e\.gender\s*=\s*'f'")},
+            {"name": "current_filter", "description": "dm.to_date='9999-01-01'", "check": _regex(r"dm\.to_date\s*=\s*'9999-01-01'")},
+            {"name": "count_distinct_mgr", "description": "COUNT(DISTINCT dm.emp_no)", "check": _regex(r"count\s*\(\s*distinct\s+dm\.emp_no")},
+            {"name": "limit_100", "description": "Ends with LIMIT 100", "check": _regex(r"limit\s+100;?\s*$")},
+        ]
+    elif category == CATEGORY_AVG_SALARY_BY_TITLE_AND_DEPT:
+        inv += [
+            {"name": "joins_salary", "description": "Join salary s", "check": _regex(r"join\s+salary")},
+            {"name": "joins_title", "description": "Join title t", "check": _regex(r"join\s+title")},
+            {"name": "joins_dept_emp", "description": "Join dept_emp de", "check": _regex(r"join\s+dept_emp")},
+            {"name": "joins_department", "description": "Join department d", "check": _regex(r"join\s+department")},
+            {"name": "current_filters", "description": "Current rows for s,de,t", "check": lambda s: all(x in s.lower() for x in ["s.to_date='9999-01-01'", "de.to_date='9999-01-01'", "t.to_date='9999-01-01'"])},
+            {"name": "avg_salary", "description": "AVG(s.salary)", "check": _regex(r"avg\s*\(\s*s\.salary")},
+            {"name": "group_by", "description": "GROUP BY dept & title", "check": _regex(r"group\s+by\s+.*(dept|title).*(title|dept)")},
+            {"name": "limit_100", "description": "Ends with LIMIT 100", "check": _regex(r"limit\s+100;?\s*$")},
+        ]
+    elif category == CATEGORY_GENDER_MANAGEMENT_COUNT:
+        inv += [
+            {"name": "joins_dept_manager", "description": "Join dept_manager dm", "check": _regex(r"join\s+dept_manager")},
+            {"name": "joins_employee", "description": "Join employee e", "check": _regex(r"join\s+employee")},
+            {"name": "current_filter", "description": "dm.to_date='9999-01-01'", "check": _regex(r"dm\.to_date\s*=\s*'9999-01-01'")},
+            {"name": "group_by_gender", "description": "GROUP BY gender", "check": _regex(r"group\s+by\s+.*gender")},
+            {"name": "count_mgr", "description": "COUNT(*) or COUNT(dm.emp_no)", "check": _regex(r"count\s*\(")},
+            {"name": "limit_100", "description": "Ends with LIMIT 100", "check": _regex(r"limit\s+100;?\s*$")},
+        ]
+    elif category == CATEGORY_SENIOR_STAFF_SUPERLATIVE:
+        inv += [
+            {"name": "joins_title", "description": "Join title t", "check": _regex(r"join\s+title")},
+            {"name": "joins_dept_emp", "description": "Join dept_emp de", "check": _regex(r"join\s+dept_emp")},
+            {"name": "joins_department", "description": "Join department d", "check": _regex(r"join\s+department")},
+            {"name": "senior_filter", "description": "t.title LIKE 'Senior%'", "check": _regex(r"t\.title\s+like\s+'senior%'")},
+            {"name": "current_filters", "description": "Current rows de.to_date & t.to_date", "check": lambda s: "de.to_date='9999-01-01'" in s.lower() and "t.to_date='9999-01-01'" in s.lower()},
+            {"name": "group_by_dept", "description": "GROUP BY d.dept_name", "check": _regex(r"group\s+by\s+.*dept_name")},
+            {"name": "order_desc", "description": "ORDER BY count desc", "check": _regex(r"order\s+by\s+.*count.*desc")},
+            {"name": "limit_100", "description": "Ends with LIMIT 100", "check": _regex(r"limit\s+100;?\s*$")},
+        ]
+    return inv
+
+# --------------- Enforcement -----------------
+
+def evaluate_invariants(sql: str, invariants: List[Dict[str, any]]) -> List[Dict[str, any]]:
+    failures = []
+    for inv in invariants:
+        try:
+            ok = inv['check'](sql)
+        except Exception:
+            ok = False
+        if not ok:
+            failures.append(inv)
+    return failures
+
+
+# More explicit prompt for join-based salary extremes queries
+REGEN_PROMPT_TEMPLATE = (
+    "You produced SQL for the user question. Some required invariants are missing.\n"
+    "Regenerate a SINGLE valid PostgreSQL SQL query that satisfies ALL listed invariants.\n"
+    "Return ONLY the SQL (no commentary). Must end with LIMIT 100; and a semicolon.\n"
+    "If the question is about highest/lowest salary by department, you MUST:\n"
+    "- Use FROM salary s\n"
+    "- JOIN dept_emp de ON s.emp_no = de.emp_no\n"
+    "- JOIN department d ON de.dept_no = d.dept_no\n"
+    "- SELECT d.dept_name, MAX(s.salary) AS highest_salary, MIN(s.salary) AS lowest_salary\n"
+    "- GROUP BY d.dept_name\n"
+    "- Use correct table aliases everywhere (no unaliased salary or dept_name)\n"
+    "- Do NOT reference columns that do not exist in the schema\n"
+    "User Question: {question}\n"
+    "Previous Attempt: {previous_sql}\n"
+    "Missing Invariants:\n{invariant_lines}\n"
+    "Regenerate now:"
+)
+
+
+def regenerate_with_constraints(question: str, previous_sql: str, missing: List[Dict[str, any]], llm, run_llm_fn, attempt: int) -> Optional[str]:
+    invariant_lines = "\n".join(f"- {m['name']}: {m['description']}" for m in missing)
+    prompt = REGEN_PROMPT_TEMPLATE.format(question=question, previous_sql=previous_sql, invariant_lines=invariant_lines)
+    # Use proper timeout values instead of hardcoded ones
+    # Start with a good timeout and decrease gradually for subsequent attempts
+    base_timeout = float(os.environ.get("INVARIANTS_BASE_TIMEOUT", 90))  # Start with 90s
+    timeout = max(30.0, base_timeout - attempt * 20.0)  # Decrease by 20s per attempt, minimum 30s
+    try:
+        raw = run_llm_fn(llm, prompt, timeout=max(8.0, timeout), retries=0)
+        # Normalize raw response to text
+        raw_text = getattr(raw, 'text', None)
+        if raw_text is None:
+            raw_text = str(raw)
+        # Extract first SELECT ... ;
+        m = re.search(r"(SELECT[\s\S]+?;)", raw_text, flags=re.IGNORECASE)
+        candidate = m.group(1) if m else raw_text.strip()
+        # Clean markdown fences
+        candidate = re.sub(r"```(?:sql)?|```", "", candidate, flags=re.IGNORECASE).strip()
+        # Ensure single statement & required LIMIT 100 at end
+        # Collapse accidental mid-statement semicolon before LIMIT
+        candidate = re.sub(r";\s*(LIMIT\s+\d+)", r" \1", candidate, flags=re.IGNORECASE)
+        # Append LIMIT 100 if missing (invariants will still verify)
+        if 'limit' not in candidate.lower():
+            if candidate.endswith(';'):
+                candidate = candidate[:-1] + ' LIMIT 100;'
+            else:
+                candidate += ' LIMIT 100;'
+        # Guarantee trailing semicolon
+        if not candidate.strip().endswith(';'):
+            candidate += ';'
+        return candidate
+    except Exception as e:
+        print(f"[INVARIANTS] Regeneration failed: {e}")
+        return None
+
+
+def enforce_invariants(question: str, sql: str, llm, run_llm_fn, max_regen: int = 2) -> Dict[str, any]:
+    category = classify_question(question)
+    if category == CATEGORY_GENERIC:
+        return {"category": category, "final_sql": sql, "regenerations": []}
+    # Dynamic schema support: the invariant catalog is purpose-built for the
+    # employee demo schema. For any other database, enforcement is bypassed
+    # and only the generic structural validation pipeline applies.
+    try:
+        try:
+            from .agentic_workflow import ACTIVE_SCHEMA_PROFILE as _asp
+        except Exception:
+            import Agent.agentic_workflow as _aw
+            _asp = getattr(_aw, "ACTIVE_SCHEMA_PROFILE", None)
+        if _asp is not None and not getattr(_asp, "is_employee_db", True):
+            print(f"[INVARIANTS] Bypassed for non-employee schema '{getattr(_asp, 'name', '?')}' (category={category})")
+            return {"category": CATEGORY_GENERIC, "final_sql": sql, "regenerations": []}
+    except Exception:
+        pass  # Profile unavailable — default legacy behavior (enforce)
+    invariants = build_invariants(category, question)
+    regenerations = []
+    current_sql = sql
+    # Increase max_regen to 5 for more LLM attempts
+    for attempt in range(max_regen + 4):
+        missing = evaluate_invariants(current_sql, invariants)
+        if not missing:
+            return {"category": category, "final_sql": current_sql, "regenerations": regenerations}
+        if attempt == max_regen:
+            # Give up, return last with missing list for transparency
+            return {"category": category, "final_sql": current_sql, "regenerations": regenerations, "remaining_missing": [m['name'] for m in missing]}
+        regen_sql = regenerate_with_constraints(question, current_sql, missing, llm, run_llm_fn, attempt)
+        if not regen_sql:
+            return {"category": category, "final_sql": current_sql, "regenerations": regenerations, "remaining_missing": [m['name'] for m in missing]}
+        regenerations.append({
+            "attempt": attempt + 1,
+            "missing_before": [m['name'] for m in missing],
+            "sql": regen_sql
+        })
+        current_sql = regen_sql
+    return {"category": category, "final_sql": current_sql, "regenerations": regenerations}

Agent/llm_health_check.py

@@ -0,0 +1,36 @@
+import os, time, json, sys
+from llama_index.llms.ollama import Ollama
+
+BASE_URL = os.environ.get("OLLAMA_BASE_URL", "http://localhost:11434")
+MODEL = os.environ.get("FINETUNED_MODEL_NAME") or os.environ.get("OLLAMA_MODEL_NAME", "gemma3:1b")
+
+print(f"Checking Ollama LLM connectivity: base_url={BASE_URL} model={MODEL}")
+
+# Lightweight client check without llama_index first (raw HTTP) to see if daemon responds
+try:
+    import requests
+    r = requests.get(f"{BASE_URL.rstrip('/')}/api/tags", timeout=5)
+    print(f"/api/tags status: {r.status_code}")
+    if r.ok:
+        tags = r.json().get('models', [])
+        names = [m.get('name') for m in tags]
+        print(f"Available models: {names}")
+        if MODEL not in names:
+            print(f"Model '{MODEL}' not yet pulled. Run: ollama pull {MODEL}")
+    else:
+        print("Non-200 response from Ollama daemon")
+except Exception as e:
+    print(f"Raw HTTP check failed: {e}")
+
+# LlamaIndex call
+try:
+    llm = Ollama(model=MODEL, base_url=BASE_URL, temperature=0.1, request_timeout=30)
+    t0 = time.time()
+    resp = llm.complete(prompt="SELECT 1;")
+    dt = time.time() - t0
+    print(f"LLM call success in {dt:.2f}s. Raw text: {resp.text[:120]!r}")
+except Exception as e:
+    print(f"LLM complete() failed: {type(e).__name__}: {e}")
+    sys.exit(1)
+
+print("Health check finished.")

Agent/mcp_client.py

@@ -0,0 +1,48 @@
+"""Minimal MCP client for local MCP server.
+
+Provides mcp_complete(prompt, timeout, context) -> str
+Tries to POST to MCP server and return the completion text. Raises on errors so caller can fallback.
+"""
+import os
+import json
+import requests
+from typing import Any, Dict, Optional
+
+MCP_URL = os.environ.get("MCP_URL", "http://127.0.0.1:8701/complete")
+
+
+def mcp_complete(prompt: str, timeout: float = 30.0, context: Optional[Dict[str, Any]] = None) -> str:
+    """Send a completion request to the local MCP server and return text.
+
+    Raises requests.RequestException on transport errors or ValueError if MCP returns error.
+    """
+    payload = {
+        "prompt": prompt,
+        "context": context or {},
+        "timeout": timeout
+    }
+    headers = {"Content-Type": "application/json"}
+    resp = requests.post(MCP_URL, data=json.dumps(payload), headers=headers, timeout=timeout)
+    resp.raise_for_status()
+    body = resp.json()
+    # Expecting {'text': '...'} from MCP server
+    if not isinstance(body, dict) or 'text' not in body:
+        raise ValueError(f"Unexpected MCP response: {body}")
+    return body['text']
+
+
+def mcp_call_tool(prompt: str, timeout: float = 60.0, context: Optional[Dict[str, Any]] = None) -> Dict[str, Any]:
+    """Call the MCP /complete endpoint and return the full JSON response.
+
+    This is useful when asking MCP to run a registered tool; the tool result will
+    appear under the `tool_result` key in the returned dict.
+    """
+    payload = {
+        "prompt": prompt,
+        "context": context or {},
+        "timeout": timeout,
+    }
+    headers = {"Content-Type": "application/json"}
+    resp = requests.post(MCP_URL, data=json.dumps(payload), headers=headers, timeout=timeout)
+    resp.raise_for_status()
+    return resp.json()

Agent/mcp_server.py

@@ -0,0 +1,630 @@
+from fastapi import FastAPI, HTTPException
+from pydantic import BaseModel
+from typing import Any, Dict, Optional, List
+import asyncio
+import importlib
+import uuid
+import os
+import re
+
+# Attempt to import agent workflow for schema/inspector & utilities (optional)
+try:
+    import Agent.agentic_workflow as aw
+    print(f"[MCP STARTUP] Successfully imported Agent.agentic_workflow")
+except Exception as e:
+    print(f"[MCP STARTUP] Failed to import Agent.agentic_workflow: {e}")
+    aw = None
+
+app = FastAPI(title="Minimal MCP Server (dev)")
+
+# Simple tool registry
+TOOLS: Dict[str, Any] = {}
+
+
+class CompleteRequest(BaseModel):
+    prompt: str
+    context: Optional[Dict[str, Any]] = None
+    timeout: Optional[float] = 30.0
+
+
+def register_tool(name: str, fn: Any):
+    TOOLS[name] = fn
+
+
+async def _call_tool(name: str, args: Dict[str, Any]) -> Any:
+    if name not in TOOLS:
+        raise KeyError(f"Tool '{name}' not registered")
+    fn = TOOLS[name]
+    if asyncio.iscoroutinefunction(fn):
+        return await fn(**(args or {}))
+    else:
+        # run sync in threadpool
+        loop = asyncio.get_running_loop()
+        return await loop.run_in_executor(None, lambda: fn(**(args or {})))
+
+
+@app.on_event("startup")
+async def _startup():
+    # Register built-in/mock tools lazily to avoid importing heavy modules at import time
+    # Try to register a real document scanner implementation; otherwise register a mock
+    ds_registered = None
+    try:
+        from .tools import document_scanner as ds
+        register_tool("document_scanner.process", ds.process_image)
+        ds_registered = f"Agent.tools.document_scanner.process_image (relative import)"
+    except Exception:
+        # If relative import fails (running as script), try absolute import
+        try:
+            import Agent.tools.document_scanner as ds2
+            register_tool("document_scanner.process", ds2.process_image)
+            ds_registered = f"Agent.tools.document_scanner.process_image (absolute import)"
+        except Exception:
+            # Register a fallback mock tool
+            def _mock_doc(**kwargs):
+                # Accept arbitrary keyword args (image_url, image_bytes, etc.) to be tolerant
+                return {"doc_id": str(uuid.uuid4()), "text": "(mock OCR output)", "metadata": {"pages": 1, "confidence": 0.0}, "received_args": kwargs}
+            register_tool("document_scanner.process", _mock_doc)
+            ds_registered = "mock(document_scanner.process)"
+
+    # Log which document scanner implementation was registered for easier debugging
+    try:
+        print(f"[MCP STARTUP] Registered document_scanner.process -> {ds_registered}")
+    except Exception:
+        pass
+
+    # Optional Groq Vision health check in cloud mode
+    try:
+        from .tools import groq_vision_health
+        hc = groq_vision_health.health_check()
+        if not hc.get("ok", True):
+            print(f"[MCP STARTUP] Groq Vision health warning: {hc.get('error')} (model={hc.get('model')})")
+        else:
+            print(f"[MCP STARTUP] Groq Vision health: ok (model={hc.get('model')})")
+    except Exception as e:
+        print(f"[MCP STARTUP] Groq Vision health check skipped: {e}")
+
+    # Register Granite Vision direct visual Q&A tool
+    gv_registered = None
+    try:
+        from .tools import granite_vision as gv
+        register_tool("granite_vision.qa", gv.qa)
+        gv_registered = "Agent.tools.granite_vision.qa (relative import)"
+    except Exception as e1:
+        print(f"[MCP STARTUP] granite_vision relative import FAILED: {type(e1).__name__}: {e1}")
+        try:
+            import Agent.tools.granite_vision as gv2
+            register_tool("granite_vision.qa", gv2.qa)
+            gv_registered = "Agent.tools.granite_vision.qa (absolute import)"
+        except Exception as e2:
+            print(f"[MCP STARTUP] granite_vision absolute import FAILED: {type(e2).__name__}: {e2}")
+            def _mock_gv(**kwargs):
+                return {"answer": "(granite_vision unavailable - mock)", "model": "mock", "received_args": kwargs}
+            register_tool("granite_vision.qa", _mock_gv)
+            gv_registered = "mock(granite_vision.qa)"
+    print(f"[MCP STARTUP] Registered granite_vision.qa -> {gv_registered}")
+
+
+    # Register OCR QA composite tool: runs OCR then runs the agent OCR QA helper
+    try:
+        # Use the global aw variable instead of re-importing
+        if not aw:
+            raise ImportError("Agent workflow not available")
+
+        async def _ocr_qa_tool(image_url: Optional[str] = None, image_bytes: Optional[bytes] = None, question: Optional[str] = None, **kwargs):
+            # Call the document scanner tool first (use the registered one to allow mocks)
+            ds_fn = TOOLS.get("document_scanner.process")
+            if not ds_fn:
+                raise RuntimeError("document_scanner.process not registered")
+
+            loop = asyncio.get_running_loop()
+            try:
+                if asyncio.iscoroutinefunction(ds_fn):
+                    tool_res = await ds_fn(image_url=image_url, image_bytes=image_bytes, options=kwargs)
+                else:
+                    tool_res = await loop.run_in_executor(None, lambda: ds_fn(image_url=image_url, image_bytes=image_bytes, options=kwargs))
+            except Exception as e:
+                return {"error": f"OCR tool failed: {e}"}
+
+            doc_id = tool_res.get("doc_id") if isinstance(tool_res, dict) else None
+            ocr_text = (tool_res.get("text") if isinstance(tool_res, dict) else str(tool_res)) or ""
+
+            # Persist OCR text so downstream callers can retrieve it via doc_id
+            try:
+                if doc_id and ocr_text:
+                    docs_dir = os.path.join("chat_store", "docs")
+                    os.makedirs(docs_dir, exist_ok=True)
+                    doc_path = os.path.join(docs_dir, f"{doc_id}.txt")
+                    # Write file (overwrite if exists)
+                    with open(doc_path, "w", encoding="utf-8") as df:
+                        df.write(ocr_text)
+                    # Also add a short history entry if agent chat store available
+                    try:
+                        if hasattr(aw, 'chat_store_private') and hasattr(aw, 'ChatMessage') and hasattr(aw, 'MessageRole'):
+                            user_msg = aw.ChatMessage(role=aw.MessageRole.USER, content=f"Uploaded document {doc_id}")
+                            assistant_msg = aw.ChatMessage(role=aw.MessageRole.ASSISTANT, content=f"OCR stored: {ocr_text[:400]}")
+                            aw.chat_store_private.add_message(key="conversation", message=user_msg)
+                            aw.chat_store_private.add_message(key="conversation", message=assistant_msg)
+                            try:
+                                aw.chat_store_private.persist(str(aw.private_store_path))
+                            except Exception:
+                                pass
+                    except Exception:
+                        pass
+            except Exception:
+                # Non-fatal: continue even if persistence fails
+                pass
+
+            # If a question was provided, answer it using the agent helper (run in executor to avoid blocking)
+            if question:
+                try:
+                    # Use executor if helper is sync to avoid blocking event loop
+                    if callable(getattr(aw, 'ocr_agent_qa', None)):
+                        answer = await loop.run_in_executor(None, lambda: aw.ocr_agent_qa(question, doc_id))
+                        return {"doc_id": doc_id, "answer": answer}
+                    else:
+                        return {"doc_id": doc_id, "answer": "ocr_agent_qa not available"}
+                except Exception as e:
+                    return {"error": f"ocr_agent_qa failed: {e}", "tool_result": tool_res}
+
+            # No question: return the OCR result and doc_id
+            return {"doc_id": doc_id, "ocr_text": ocr_text}
+
+        register_tool("ocr_qa", _ocr_qa_tool)
+        print(f"[MCP STARTUP] Registered ocr_qa -> real _ocr_qa_tool (agent available)")
+    except Exception:
+        # If agent import fails, register a tolerant mock tool
+        def _ocr_qa_mock(**kwargs):
+            return {"doc_id": str(uuid.uuid4()), "answer": "(ocr_qa unavailable - mock response)", "received_args": kwargs}
+        register_tool("ocr_qa", _ocr_qa_mock)
+        print(f"[MCP STARTUP] Registered ocr_qa -> mock(ocr_qa)")
+
+    # ------------------------------------------------------------------
+    # Deterministic schema-aware SQL helper tools (agentic compliant)
+    # ------------------------------------------------------------------
+    ENABLE_FAST_SQL = os.environ.get("ENABLE_MCP_FAST_TOOLS", "1") in ("1","true","True")
+    if not ENABLE_FAST_SQL:
+        print("[MCP STARTUP] Fast SQL tools disabled via ENABLE_MCP_FAST_TOOLS=0")
+        return
+
+    # Initialize database connection if not already done
+    if not aw:
+        print("[MCP STARTUP][FAST] Agent workflow not available; skipping fast SQL tool registration.")
+        return
+        
+    if not getattr(aw, 'inspector', None) or not getattr(aw, 'available_tables', None):
+        print("[MCP STARTUP][FAST] Database not initialized; attempting to initialize...")
+        try:
+            # Initialize database connection - use await since we're in async context.
+            # Priority chain mirrors worker startup: 1) env DB  2) active schema
+            # profile from the registry  3) bundled demo employee SQLite file.
+            db_uri = os.environ.get("DB_CONNECTION_URL")
+            if not db_uri:
+                demo_db_path = os.path.join(
+                    os.path.dirname(os.path.dirname(os.path.abspath(__file__))),
+                    "employee-sample-database", "sqlite", "dataset_small", "employee.db",
+                )
+                try:
+                    from .schema_profile import profile_registry as _reg
+                    _active = _reg.active() or {}
+                    db_uri = _active.get("db_url")
+                except Exception:
+                    db_uri = None
+                if not db_uri and os.path.isfile(demo_db_path):
+                    db_uri = f"sqlite:///{demo_db_path.replace(os.sep, '/')}"
+                    print("[MCP STARTUP][FAST] Using bundled demo employee database.")
+            if not db_uri:
+                print("[MCP STARTUP][FAST] No DB env, active profile, or bundled demo found; skipping fast SQL tool registration.")
+                return
+            db_result = await aw.initialize_database(db_uri)
+            if db_result.get("tables"):
+                print(f"[MCP STARTUP][FAST] Database initialized with {len(db_result['tables'])} tables")
+            else:
+                print("[MCP STARTUP][FAST] Database initialization failed; skipping fast SQL tool registration.")
+                return
+        except Exception as db_err:
+            print(f"[MCP STARTUP][FAST] Database initialization error: {db_err}; skipping fast SQL tool registration.")
+            return
+
+    inspector = aw.inspector
+    available_tables = aw.available_tables
+
+    def _colnames(table: str) -> List[str]:
+        try:
+            return [c['name'] for c in inspector.get_columns(table_name=table)]
+        except Exception:
+            return []
+
+    def _has_tables(*tables: str) -> bool:
+        at_lower = {t.lower() for t in available_tables}
+        return all(t.lower() in at_lower for t in tables)
+
+    # Simple validation & sanitization re-use
+    validate_sql_query = getattr(aw, 'validate_sql_query', lambda q: {"valid": True})
+    fix_common_sql_errors = getattr(aw, 'fix_common_sql_errors', lambda q: q)
+
+    async def _verify_sql(sql: str, question: str, tag: str) -> str:
+        """Optional light LLM verification (short prompt) if LLM available."""
+        if not aw or not getattr(aw, 'Settings', None) or not getattr(aw.Settings, 'llm', None):
+            return sql
+        try:
+            _schema_block = ""
+            try:
+                if aw.ACTIVE_SCHEMA_PROFILE.is_employee_db:
+                    _schema_block = aw.EMPLOYEE_DB_SCHEMA
+                else:
+                    _schema_block = aw.ACTIVE_SCHEMA_PROFILE.build_digest(max_tables=10)
+            except Exception:
+                _schema_block = ""
+            prompt = (
+                (f"{_schema_block}\n\n" if _schema_block else "")
+                + "You are an SQL verifier."
+                + f" Tag={tag}."
+                + " If the candidate SQL is valid and uses only existing tables/columns keep it."
+                " If SMALL fixes (alias, LIMIT 100, semicolon) needed, fix them. Never invent tables/columns."
+                f"\nQuestion: {question}\nCandidate SQL:\n{sql}\nReturn ONLY final SQL." )
+            resp = aw.Settings.llm.complete(prompt=prompt)
+            txt = getattr(resp, 'text', str(resp)).strip()
+            if '```' in txt:
+                parts = re.split(r"```(?:sql)?", txt, flags=re.IGNORECASE)
+                txt = '\n'.join(p for p in parts if 'select' in p.lower() or 'with' in p.lower())
+            if txt.lower().startswith('sql'):
+                txt = txt[3:].strip()
+            txt = fix_common_sql_errors(txt)
+            v = validate_sql_query(txt)
+            if v.get('valid'):
+                return txt
+        except Exception as ve:
+            print(f"[FAST][VERIFY][WARN] Verification failed: {ve}; using original SQL")
+        return sql
+
+    def _build_sql(select_cols: List[str], base_table: str, joins: Optional[List[str]] = None,
+                   where: Optional[List[str]] = None, group_by: Optional[List[str]] = None,
+                   order_by: Optional[str] = None, limit: Optional[int] = 100) -> str:
+        parts = ["SELECT " + ", ".join(select_cols), "FROM " + base_table]
+        for j in (joins or []):
+            parts.append(j)
+        if where:
+            parts.append("WHERE " + " AND ".join(where))
+        if group_by:
+            parts.append("GROUP BY " + ", ".join(group_by))
+        if order_by:
+            parts.append("ORDER BY " + order_by)
+        if limit is not None:
+            parts.append(f"LIMIT {limit}")
+        sql = " ".join(parts)
+        if not sql.strip().endswith(';'):
+            sql += ';'
+        return sql
+
+    async def tool_list_departments(question: str = ""):
+        if not _has_tables('department'):
+            return {"error": "department table not available"}
+        cols = _colnames('department')
+        name_col = 'dept_name' if 'dept_name' in cols else (cols[0] if cols else 'dept_name')
+        sql = _build_sql([f"d.{name_col} AS department_name"], 'department d', order_by='department_name ASC')
+        sql = fix_common_sql_errors(sql)
+        sql = await _verify_sql(sql, question or 'List departments', 'list_departments')
+        return {"sql": sql}
+
+    async def tool_count_employees(question: str = ""):
+        if not _has_tables('employee'):
+            return {"error": "employee table not available"}
+        sql = _build_sql(["COUNT(*) AS total_employees"], 'employee e', limit=None)
+        sql = fix_common_sql_errors(sql)
+        sql = await _verify_sql(sql, question or 'Total employees', 'count_employees')
+        return {"sql": sql}
+
+    async def tool_employee_count_by_department(question: str = ""):
+        if not _has_tables('department','dept_emp'):
+            return {"error": "required tables missing"}
+        dcols = _colnames('department'); decols = _colnames('dept_emp')
+        dept_name = 'dept_name' if 'dept_name' in dcols else (dcols[0] if dcols else 'dept_name')
+        dept_no = 'dept_no' if 'dept_no' in dcols or 'dept_no' in decols else 'dept_no'
+        emp_no = 'emp_no' if 'emp_no' in decols else 'emp_no'
+        where = []
+        if 'to_date' in decols:
+            where.append("de.to_date = '9999-01-01'")
+        sql = _build_sql([f"d.{dept_name} AS department_name", f"COUNT(DISTINCT de.{emp_no}) AS employee_count"],
+                         'department d', joins=[f"JOIN dept_emp de ON d.{dept_no} = de.{dept_no}"],
+                         where=where, group_by=[f"d.{dept_name}"], order_by='employee_count DESC')
+        sql = fix_common_sql_errors(sql)
+        sql = await _verify_sql(sql, question or 'Employees per department', 'employee_count_by_department')
+        return {"sql": sql}
+
+    async def tool_gender_pay_gap(question: str = ""):
+        if not _has_tables('department','dept_emp','employee','salary'):
+            return {"error": "required tables missing"}
+        scolumns = _colnames('salary')
+        sval = 'salary' if 'salary' in scolumns else ('amount' if 'amount' in scolumns else (scolumns[0] if scolumns else 'salary'))
+        where = ["de.to_date = '9999-01-01'", "s.to_date = '9999-01-01'"]
+        sql = _build_sql([
+            "d.dept_name",
+            f"AVG(CASE WHEN e.gender='M' THEN s.{sval} END) AS avg_male_salary",
+            f"AVG(CASE WHEN e.gender='F' THEN s.{sval} END) AS avg_female_salary",
+            f"(AVG(CASE WHEN e.gender='M' THEN s.{sval} END) - AVG(CASE WHEN e.gender='F' THEN s.{sval} END)) AS gap",
+            f"CASE WHEN AVG(CASE WHEN e.gender='M' THEN s.{sval} END)=0 THEN NULL ELSE ((AVG(CASE WHEN e.gender='M' THEN s.{sval} END) - AVG(CASE WHEN e.gender='F' THEN s.{sval} END))/NULLIF(AVG(CASE WHEN e.gender='M' THEN s.{sval} END),0))*100 END AS pct_gap"
+        ], 'department d',
+            joins=['JOIN dept_emp de ON d.dept_no = de.dept_no','JOIN employee e ON de.emp_no = e.emp_no','JOIN salary s ON e.emp_no = s.emp_no'],
+            where=where, group_by=['d.dept_name'], order_by='gap DESC')
+        sql = fix_common_sql_errors(sql)
+        sql = await _verify_sql(sql, question or 'Gender pay gap', 'gender_pay_gap')
+        return {"sql": sql}
+
+    async def tool_salary_extremes(question: str = ""):
+        if not _has_tables('salary'):
+            return {"error": "salary table missing"}
+        scolumns = _colnames('salary'); sval = 'salary' if 'salary' in scolumns else ('amount' if 'amount' in scolumns else (scolumns[0] if scolumns else 'salary'))
+        where = ["s.to_date = '9999-01-01'"] if 'to_date' in scolumns else []
+        sql = _build_sql([f"MAX(s.{sval}) AS highest_salary", f"MIN(s.{sval}) AS lowest_salary"], 'salary s', where=where, limit=None)
+        sql = fix_common_sql_errors(sql)
+        sql = await _verify_sql(sql, question or 'Salary extremes', 'salary_extremes')
+        return {"sql": sql}
+
+    async def tool_salary_range_department(department_name: str = "Development", question: str = ""):
+        if not _has_tables('department','dept_emp','salary'):
+            return {"error": "required tables missing"}
+        scolumns = _colnames('salary'); sval = 'salary' if 'salary' in scolumns else ('amount' if 'amount' in scolumns else (scolumns[0] if scolumns else 'salary'))
+        # Escape single quotes in department_name for SQL safety (basic)
+        escaped_dept = department_name.replace("'", "''")
+        where = ["de.to_date = '9999-01-01'", "s.to_date = '9999-01-01'", f"d.dept_name = '{escaped_dept}'"]
+        sql = _build_sql(["d.dept_name", f"MAX(s.{sval}) AS max_salary", f"MIN(s.{sval}) AS min_salary"], 'department d',
+                         joins=['JOIN dept_emp de ON d.dept_no = de.dept_no','JOIN salary s ON de.emp_no = s.emp_no'],
+                         where=where, group_by=['d.dept_name'], limit=None)
+        sql = fix_common_sql_errors(sql)
+        sql = await _verify_sql(sql, question or 'Department salary range', 'salary_range_department')
+        return {"sql": sql, "department": department_name}
+
+    async def tool_top_paid_employees(top_n: int = 10, question: str = ""):
+        if not _has_tables('employee','salary'):
+            return {"error": "required tables missing"}
+        scolumns = _colnames('salary'); sval = 'salary' if 'salary' in scolumns else (scolumns[0] if scolumns else 'salary')
+        where = ["s.to_date = '9999-01-01'"] if 'to_date' in scolumns else []
+        sql = _build_sql(["e.emp_no","e.first_name","e.last_name", f"s.{sval} AS salary"], 'employee e',
+                         joins=['JOIN salary s ON e.emp_no = s.emp_no'], where=where,
+                         order_by=f"s.{sval} DESC", limit=top_n)
+        sql = fix_common_sql_errors(sql)
+        sql = await _verify_sql(sql, question or 'Top paid employees', 'top_paid_employees')
+        return {"sql": sql, "top_n": top_n}
+
+    async def tool_department_highest_avg_salary(question: str = ""):
+        if not _has_tables('department','dept_emp','salary'):
+            return {"error": "required tables missing"}
+        scolumns = _colnames('salary'); sval = 'salary' if 'salary' in scolumns else (scolumns[0] if scolumns else 'salary')
+        where = ["de.to_date = '9999-01-01'", "s.to_date = '9999-01-01'"]
+        sql = _build_sql(["d.dept_name", f"AVG(s.{sval}) AS avg_salary"], 'department d',
+                         joins=['JOIN dept_emp de ON d.dept_no = de.dept_no','JOIN salary s ON de.emp_no = s.emp_no'],
+                         where=where, group_by=['d.dept_name'], order_by='avg_salary DESC', limit=1)
+        sql = fix_common_sql_errors(sql)
+        sql = await _verify_sql(sql, question or 'Department highest average salary', 'dept_highest_avg_salary')
+        return {"sql": sql}
+
+    async def tool_department_manager_listing(question: str = ""):
+        if not _has_tables('department','dept_manager','employee'):
+            return {"error": "required tables missing"}
+        where = ["dm.to_date = '9999-01-01'"]
+        sql = _build_sql(["d.dept_name AS department_name","e.first_name","e.last_name"], 'department d',
+                         joins=['JOIN dept_manager dm ON d.dept_no = dm.dept_no','JOIN employee e ON dm.emp_no = e.emp_no'],
+                         where=where, order_by='department_name ASC')
+        sql = fix_common_sql_errors(sql)
+        sql = await _verify_sql(sql, question or 'Department managers', 'department_manager_listing')
+        return {"sql": sql}
+
+    # NEW: Complex analytical tools for business intelligence
+    async def tool_gender_distribution_by_dept(question: str = ""):
+        if not _has_tables('department','dept_emp','employee'):
+            return {"error": "required tables missing"}
+        where = ["de.to_date = '9999-01-01'"]
+        sql = _build_sql([
+            "d.dept_name AS department_name",
+            "e.gender",
+            "COUNT(*) AS count",
+            "ROUND(COUNT(*) * 100.0 / SUM(COUNT(*)) OVER (PARTITION BY d.dept_name), 2) AS percentage"
+        ], 'department d',
+            joins=['JOIN dept_emp de ON d.dept_no = de.dept_no','JOIN employee e ON de.emp_no = e.emp_no'],
+            where=where, group_by=['d.dept_name', 'e.gender'], order_by='department_name, gender')
+        sql = fix_common_sql_errors(sql)
+        sql = await _verify_sql(sql, question or 'Gender distribution by department', 'gender_distribution_by_dept')
+        return {"sql": sql}
+
+    async def tool_hiring_trend_by_year(question: str = ""):
+        if not _has_tables('employee'):
+            return {"error": "required table employee missing"}
+        sql = _build_sql([
+            "EXTRACT(YEAR FROM e.hire_date) AS hire_year",
+            "COUNT(*) AS employees_hired",
+            "COUNT(CASE WHEN e.gender='M' THEN 1 END) AS male_hires",
+            "COUNT(CASE WHEN e.gender='F' THEN 1 END) AS female_hires"
+        ], 'employee e', group_by=['EXTRACT(YEAR FROM e.hire_date)'], order_by='hire_year ASC')
+        sql = fix_common_sql_errors(sql)
+        sql = await _verify_sql(sql, question or 'Hiring trends by year', 'hiring_trend_by_year')
+        return {"sql": sql}
+
+    async def tool_employee_tenure_by_dept(question: str = ""):
+        if not _has_tables('department','dept_emp','employee'):
+            return {"error": "required tables missing"}
+        where = ["de.to_date = '9999-01-01'"]
+        sql = _build_sql([
+            "d.dept_name AS department_name",
+            "AVG(EXTRACT(DAYS FROM (CURRENT_DATE - e.hire_date))/365.25) AS avg_tenure_years",
+            "MIN(EXTRACT(DAYS FROM (CURRENT_DATE - e.hire_date))/365.25) AS min_tenure_years", 
+            "MAX(EXTRACT(DAYS FROM (CURRENT_DATE - e.hire_date))/365.25) AS max_tenure_years"
+        ], 'department d',
+            joins=['JOIN dept_emp de ON d.dept_no = de.dept_no','JOIN employee e ON de.emp_no = e.emp_no'],
+            where=where, group_by=['d.dept_name'], order_by='avg_tenure_years DESC')
+        sql = fix_common_sql_errors(sql)
+        sql = await _verify_sql(sql, question or 'Employee tenure by department', 'employee_tenure_by_dept')
+        return {"sql": sql}
+
+    async def tool_female_managers_count(question: str = ""):
+        if not _has_tables('dept_manager','employee'):
+            return {"error": "required tables missing"}
+        where = ["dm.to_date = '9999-01-01'", "e.gender = 'F'"]
+        sql = _build_sql([
+            "COUNT(*) AS female_managers_count",
+            "COUNT(*) * 100.0 / (SELECT COUNT(*) FROM dept_manager dm2 WHERE dm2.to_date = '9999-01-01') AS percentage_female_managers"
+        ], 'dept_manager dm',
+            joins=['JOIN employee e ON dm.emp_no = e.emp_no'],
+            where=where, limit=None)
+        sql = fix_common_sql_errors(sql)
+        sql = await _verify_sql(sql, question or 'Female managers count', 'female_managers_count')
+        return {"sql": sql}
+
+    async def tool_avg_salary_by_title_dept(question: str = ""):
+        """Get average salary by title and department."""
+        if not _has_tables('department', 'dept_emp', 'salary', 'title', 'employee'):
+            return {"error": "required tables missing"}
+        where = ["de.to_date = '9999-01-01'", "s.to_date = '9999-01-01'", "t.to_date = '9999-01-01'"]
+        sql = _build_sql(["d.dept_name AS department_name", "t.title", "AVG(s.salary) AS avg_salary"], 'department d',
+                         joins=['JOIN dept_emp de ON d.dept_no = de.dept_no',
+                               'JOIN employee e ON de.emp_no = e.emp_no',
+                               'JOIN salary s ON e.emp_no = s.emp_no',
+                               'JOIN title t ON e.emp_no = t.emp_no'],
+                         where=where, group_by=['d.dept_name', 't.title'], 
+                         order_by='d.dept_name, avg_salary DESC', limit=100)
+        sql = fix_common_sql_errors(sql)
+        sql = await _verify_sql(sql, question or 'Average salary by title and department', 'avg_salary_by_title_dept')
+        return {"sql": sql}
+
+    # ---- Registration ----
+    # Dynamic schema support: the 14 fast_sql.* tools are purpose-built SQL
+    # skeletons for the employee demo schema. Register them only when the
+    # active profile IS the employee demo. Otherwise, register the generic
+    # schema-grounded tools that work on ANY database.
+    is_employee_profile = bool(
+        getattr(aw, 'ACTIVE_SCHEMA_PROFILE', None)
+        and aw.ACTIVE_SCHEMA_PROFILE.is_employee_db
+    )
+
+    if is_employee_profile:
+        register_tool('fast_sql.list_departments', tool_list_departments)
+        register_tool('fast_sql.count_employees', tool_count_employees)
+        register_tool('fast_sql.employee_count_by_department', tool_employee_count_by_department)
+        register_tool('fast_sql.gender_pay_gap', tool_gender_pay_gap)  # RE-ENABLED: Complex analytical queries
+        register_tool('fast_sql.salary_extremes', tool_salary_extremes)
+        register_tool('fast_sql.salary_range_department', tool_salary_range_department)
+        register_tool('fast_sql.top_paid_employees', tool_top_paid_employees)
+        register_tool('fast_sql.department_highest_avg_salary', tool_department_highest_avg_salary)
+        register_tool('fast_sql.department_manager_listing', tool_department_manager_listing)
+
+        # Register new complex analytical tools
+        register_tool('fast_sql.gender_distribution_by_dept', tool_gender_distribution_by_dept)
+        register_tool('fast_sql.hiring_trend_by_year', tool_hiring_trend_by_year)
+        register_tool('fast_sql.employee_tenure_by_dept', tool_employee_tenure_by_dept)
+        register_tool('fast_sql.female_managers_count', tool_female_managers_count)
+        register_tool('fast_sql.avg_salary_by_title_dept', tool_avg_salary_by_title_dept)
+        print('[MCP STARTUP] Registered employee-demo fast SQL tools: '
+              + ', '.join([k for k in TOOLS if k.startswith('fast_sql.')]))
+    else:
+        # ---------- Generic schema-grounded tools (any database) ----------
+        def tool_schema_list_tables(**kwargs):
+            return {"tables": list(available_tables)}
+
+        def tool_schema_describe_table(table: str = "", **kwargs):
+            if not table:
+                return {"error": "table is required", "tables": list(available_tables)}
+            cols = _colnames(table)
+            if not cols:
+                # Case-insensitive retry
+                lower_map = {t.lower(): t for t in available_tables}
+                table = lower_map.get(table.lower())
+                if table:
+                    cols = _colnames(table)
+            return {"table": table, "exists": bool(cols), "columns": cols}
+
+        def tool_schema_sample_data(table: str = "", limit: int = 3, **kwargs):
+            if not table:
+                return {"error": "table is required"}
+            try:
+                from sqlalchemy import text as _sa_text
+                safe_limit = max(1, min(int(limit), 10))
+                with aw.sql_database.connect() as conn:
+                    rows = conn.execute(
+                        _sa_text(f'SELECT * FROM "{table}" LIMIT {safe_limit}')
+                    ).mappings().all()
+                    return {"table": table, "rows": [dict(r) for r in rows]}
+            except Exception:
+                try:
+                    from sqlalchemy import text as _sa_text
+                    safe_limit = max(1, min(int(limit), 10))
+                    with aw.sql_database.connect() as conn:
+                        rows = conn.execute(
+                            _sa_text(f"SELECT * FROM {table} LIMIT {safe_limit}")
+                        ).mappings().all()
+                        return {"table": table, "rows": [dict(r) for r in rows]}
+                except Exception as e2:
+                    return {"table": table, "error": str(e2)}
+
+        async def tool_sql_build_select(select_cols: Optional[List[str]] = None, base_table: str = "",
+                                        joins: Optional[List[str]] = None, where: Optional[List[str]] = None,
+                                        group_by: Optional[List[str]] = None, order_by: Optional[str] = None,
+                                        limit: Optional[int] = 100, question: str = "", **kwargs):
+            """Deterministic schema-grounded SELECT builder: validates requested
+            columns against the live table, uses SQLAlchemy inspector, then an
+            optional light LLM refinement. Returns a validated SQL hint."""
+            cols = _colnames(base_table)
+            if not cols:
+                lower_map = {t.lower(): t for t in available_tables}
+                base_table = lower_map.get((base_table or "").lower(), base_table)
+                cols = _colnames(base_table)
+            if not cols:
+                return {"error": f"Unknown table '{base_table}'",
+                        "tables": list(available_tables)}
+            _cols_lower = {str(c).lower(): str(c) for c in cols}
+            for raw in (select_cols or []):
+                # tokens like alias.col, col, COUNT(col) etc.
+                for tok in re.findall(r"[A-Za-z_][\w]*(?:\.[A-Za-z_]\w*)?", str(raw)):
+                    t_, sep, c_ = tok.partition('.')
+                    if sep and c_.lower() not in _cols_lower and c_.lower() != '1':
+                        return {"error": f"Unknown column '{tok}' in table '{base_table}'",
+                                "columns": cols,
+                                "hint": "select base_table.column for aggregate args, e.g. COUNT(col)"}
+            sql = _build_sql(select_cols=select_cols, base_table=base_table, joins=joins,
+                             where=where, group_by=group_by, order_by=order_by, limit=limit)
+            sql = fix_common_sql_errors(sql)
+            sql = await _verify_sql(sql, question or f"SELECT from {base_table}", f"build_select_{base_table}")
+            return {"sql": sql, "table": base_table, "columns": cols}
+
+        register_tool('schema.list_tables', tool_schema_list_tables)
+        register_tool('schema.describe_table', tool_schema_describe_table)
+        register_tool('schema.sample_data', tool_schema_sample_data)
+        register_tool('sql.build_select', tool_sql_build_select)
+        print('[MCP STARTUP] Registered GENERIC schema tools: '
+              + ', '.join([k for k in TOOLS if k.startswith('schema.') or k.startswith('sql.')]))
+        print('[MCP STARTUP] Skipping employee-demo fast_sql.* tools (non-employee schema active)')
+
+
+
+@app.post("/complete")
+async def complete(req: CompleteRequest):
+    """Minimal completion endpoint:
+    - If context contains {'tool_call': {'name': ..., 'args': {...}}}, the server will run the tool and return its result.
+    - Otherwise it returns a simple echo of the prompt under 'text'.
+    This scaffold is purposely small so you can extend tool orchestration later.
+    """
+    ctx = req.context or {}
+    # Tool invocation flow
+    if 'tool_call' in ctx:
+        tc = ctx['tool_call']
+        name = tc.get('name')
+        args = tc.get('args', {})
+        try:
+            tool_res = await _call_tool(name, args)
+            return {"text": f"Tool '{name}' executed.", "tool_result": tool_res}
+        except KeyError as ke:
+            raise HTTPException(status_code=404, detail=str(ke))
+        except Exception as e:
+            raise HTTPException(status_code=500, detail=f"Tool call failed: {e}")
+
+    # No tool requested: return a simple completion echo for now
+    # Future: integrate a local model here and tool loop orchestration
+    return {"text": req.prompt}
+
+
+@app.get("/tools")
+def list_tools():
+    return {"tools": list(TOOLS.keys())}
+
+
+@app.get("/health")
+def health():
+    return {"status": "ok", "tools_registered": len(TOOLS)}

Agent/memory_manager.py

@@ -0,0 +1,467 @@
+"""
+Memory management utilities for the text2sql agent.
+"""
+import gc
+import os
+import time
+import psutil
+import threading
+import weakref
+from contextlib import contextmanager
+from typing import Optional, Callable, Dict, Any
+
+class MemoryManager:
+    """Memory manager for monitoring and managing system memory."""
+    
+    def __init__(self, threshold_mb: float = 1000.0, critical_mb: float = 300.0,
+                 check_interval: float = 0.5, monitor_active: bool = True):
+        """Initialize the memory manager.
+
+        Args:
+            threshold_mb: Memory threshold in MB below which to take action
+            critical_mb: Critical memory level in MB below which to abort operations
+            check_interval: Interval in seconds for background memory checks
+            monitor_active: Whether to start the background monitor thread
+        """
+        # Allow environment overrides to tune aggressiveness
+        try:
+            _env_threshold = os.environ.get("MEM_THRESHOLD_MB")
+            _env_critical = os.environ.get("MEM_CRITICAL_MB")
+            if _env_threshold: threshold_mb = float(_env_threshold)
+            if _env_critical: critical_mb = float(_env_critical)
+        except Exception:
+            pass
+        self.threshold_mb = threshold_mb
+        self.critical_mb = critical_mb
+        self.check_interval = check_interval
+        self._lock = threading.RLock()
+        self._monitor_thread = None
+        self._keep_monitoring = False
+
+        # Track instances with weak references when possible to avoid holding
+        # strong references that prevent GC. Keep a fallback list for objects
+        # that are not weak-referenceable.
+        self._active_llm_refs = []  # list of weakref.ref where possible
+        self._active_llm_strong = []  # fallback strong refs
+        self._instance_count = 0
+
+        # Soft cap to avoid accidentally registering many parallel LLMs
+        self.max_instances = 1
+
+        # Margin (MB) above critical required to allow new registrations
+        self.registration_margin_mb = 200.0
+
+        # Track last emergency cleanup time to avoid repeated rapid-fire releases
+        self._last_emergency = 0.0
+        self._emergency_cooldown = 10.0  # seconds between emergency attempts
+        # Counter-based suspension flag to prevent emergency releases while in critical sections
+        self._suspend_emergency_depth = 0
+
+        # Minimum available memory (MB) at which we will STILL retain a single LLM instance
+        # even if below critical, to avoid thrashing (re-init cost is high). Configurable.
+        try:
+            self.min_retain_mb = float(os.environ.get("MIN_RETAIN_LLM_MB", "250"))
+        except Exception:
+            self.min_retain_mb = 250.0
+
+        # Start memory monitor if requested
+        if monitor_active:
+            self.start_monitor()
+
+    def _prune_active_instances(self):
+        """Remove dead weakrefs and return list of live instances."""
+        live = []
+        # Prune weak refs
+        for ref in list(self._active_llm_refs):
+            inst = ref()
+            if inst is None:
+                try:
+                    self._active_llm_refs.remove(ref)
+                except ValueError:
+                    pass
+            else:
+                live.append(inst)
+        # Add strong refs
+        for inst in list(self._active_llm_strong):
+            if inst is not None:
+                live.append(inst)
+        return live
+    
+    def register_llm_instance(self, llm_instance):
+        """Register an LLM instance for tracking.
+
+        This will refuse registration (and attempt to release the provided
+        instance) if the system memory is too low. It prefers weak references
+        so that the manager itself does not keep objects alive.
+        """
+        with self._lock:
+            # Prune dead weakrefs first
+            self._prune_active_instances()
+
+            stats = self.get_memory_stats()
+            # Relax policy: always allow the FIRST instance as long as we are above critical.
+            # Only enforce margin for additional instances (even though max_instances is 1 today).
+            if self._instance_count == 0:
+                if stats['available_mb'] < self.critical_mb:
+                    # Too dangerous even for first instance
+                    try:
+                        if hasattr(llm_instance, 'close'):
+                            llm_instance.close()
+                        elif hasattr(llm_instance, 'release'):
+                            llm_instance.release()
+                    except Exception:
+                        pass
+                    raise MemoryError(f"Refusing first LLM registration: below critical ({stats['available_mb']:.1f}MB < {self.critical_mb}MB)")
+            else:
+                # For subsequent instances keep the stricter margin
+                if stats['available_mb'] < (self.critical_mb + self.registration_margin_mb):
+                    try:
+                        if hasattr(llm_instance, 'close'):
+                            llm_instance.close()
+                        elif hasattr(llm_instance, 'release'):
+                            llm_instance.release()
+                    except Exception:
+                        pass
+                    raise MemoryError(f"Refusing to register additional LLM (avail={stats['available_mb']:.1f}MB; need >= {self.critical_mb + self.registration_margin_mb:.1f}MB)")
+
+            # Enforce max instances cap
+            if self._instance_count >= self.max_instances:
+                raise MemoryError(f"Refusing to register LLM instance: max_instances={self.max_instances} reached")
+
+            # Try to create a weakref to avoid holding a strong reference
+            try:
+                ref = weakref.ref(llm_instance, lambda r: None)
+                self._active_llm_refs.append(ref)
+            except TypeError:
+                # Object not weak-referenceable; store strong ref but warn
+                self._active_llm_strong.append(llm_instance)
+
+            self._instance_count += 1
+            print(f"LLM instance registered (total: {self._instance_count})")
+    
+    def unregister_llm_instance(self, llm_instance):
+        """Unregister an LLM instance from tracking."""
+        with self._lock:
+            # Remove from weak refs
+            removed = False
+            for ref in list(self._active_llm_refs):
+                inst = ref()
+                if inst is None:
+                    # Dead ref; prune
+                    try:
+                        self._active_llm_refs.remove(ref)
+                    except ValueError:
+                        pass
+                    continue
+                if inst is llm_instance:
+                    try:
+                        self._active_llm_refs.remove(ref)
+                    except ValueError:
+                        pass
+                    removed = True
+                    break
+
+            # Remove from strong refs
+            if not removed:
+                for i, instance in enumerate(list(self._active_llm_strong)):
+                    if instance is llm_instance:
+                        del self._active_llm_strong[i]
+                        removed = True
+                        break
+
+            if removed:
+                self._instance_count = max(0, self._instance_count - 1)
+                print(f"LLM instance unregistered (remaining: {self._instance_count})")
+    
+    def get_memory_stats(self) -> Dict[str, float]:
+        """Get current memory statistics."""
+        vm = psutil.virtual_memory()
+        return {
+            "total_mb": vm.total / (1024 * 1024),
+            "available_mb": vm.available / (1024 * 1024),
+            "used_mb": vm.used / (1024 * 1024),
+            "percent_used": vm.percent
+        }
+    
+    def is_memory_safe(self) -> bool:
+        """Check if memory is at a safe level."""
+        return self.get_memory_stats()["available_mb"] >= self.threshold_mb
+    
+    def is_memory_critical(self) -> bool:
+        """Check if memory is at a critical level."""
+        stats = self.get_memory_stats()
+        available = stats["available_mb"]
+        is_critical = available <= self.critical_mb
+        
+        # Debug logging for memory state
+        if is_critical:
+            print(f"🚨 [MEMORY_DEBUG] Critical state: {available:.1f}MB <= {self.critical_mb}MB threshold")
+        
+        return is_critical
+    
+    def force_collect_garbage(self):
+        """Force aggressive garbage collection."""
+        # Run multiple collection cycles
+        for i in range(3):
+            gc.collect(i)
+        
+        # Clear any reference cycles
+        if hasattr(gc, 'freeze'):
+            gc.freeze()
+        
+        # Clear module caches if possible
+        if hasattr(gc, 'clear_caches'):
+            gc.clear_caches()
+            
+        # Log memory status after cleanup
+        stats = self.get_memory_stats()
+        #print(f"After garbage collection: {stats['available_mb']:.2f}MB available ({stats['percent_used']}% used)")
+    
+    def _monitor_memory(self):
+        """Background thread for memory monitoring."""
+        while self._keep_monitoring:
+            stats = self.get_memory_stats()
+            
+            # Check for critical memory conditions
+            if stats["available_mb"] <= self.critical_mb:
+                #print(f"⚠️ CRITICAL MEMORY WARNING: Only {stats['available_mb']:.2f}MB available!")
+                self.force_collect_garbage()
+                
+                # If still critical after cleanup, take more drastic measures
+                if self.is_memory_critical():
+                    now = time.time()
+                    # Only attempt emergency release if cooldown passed
+                    if now - self._last_emergency > self._emergency_cooldown:
+                        # Check if we have tracked instances (prune dead refs first)
+                        live = self._prune_active_instances()
+                        if live:
+                            # If only a single LLM instance and we still have above min_retain_mb, skip releasing to prevent constant re-init.
+                            if len(live) == 1 and stats["available_mb"] > self.min_retain_mb:
+                                print(f"⚠️ Critical memory but retaining single LLM (avail={stats['available_mb']:.1f}MB > {self.min_retain_mb}MB safeguard)")
+                            else:
+                                # Skip emergency release while suspended
+                                if self._suspend_emergency_depth > 0:
+                                    print(f"🚫 Emergency release suppressed (depth={self._suspend_emergency_depth}) during critical section")
+                                else:
+                                    self._emergency_release_llm_instances()
+                        else:
+                            # Log once per cooldown when no instances are registered
+                            print("🚨 Emergency memory state detected but no LLM instances registered to release")
+                        self._last_emergency = now
+                    else:
+                        # Cooldown active; skip repeated emergency attempts
+                        pass
+            
+            # Regular check for low memory
+            elif stats["available_mb"] <= self.threshold_mb:
+                #print(f"⚠️ Low memory warning: {stats['available_mb']:.2f}MB available")
+                self.force_collect_garbage()
+            
+            # Sleep for the check interval
+            time.sleep(self.check_interval)
+    
+    def _emergency_release_llm_instances(self):
+        """Emergency release of LLM instances to free memory."""
+        with self._lock:
+            # Build list of live instances from weak and strong refs
+            instances = self._prune_active_instances()
+            print(f"Attempting to release {len(instances)} LLM instances")
+
+            for llm in instances:
+                try:
+                    # Try to close or release resources
+                    if hasattr(llm, 'close'):
+                        llm.close()
+                    elif hasattr(llm, 'release'):
+                        llm.release()
+                    
+                    # Set to None to break references
+                    if hasattr(llm, '_client'):
+                        llm._client = None
+                    if hasattr(llm, '_model'):
+                        llm._model = None
+                except Exception as e:
+                    print(f"Error releasing LLM instance: {str(e)}")
+            
+            # Clear our tracking lists
+            self._active_llm_refs.clear()
+            self._active_llm_strong.clear()
+            self._instance_count = 0
+
+            # Force garbage collection after cleanup and attempt to return pages
+            self.force_collect_garbage()
+            try:
+                # Best-effort working set trim to return pages to OS (Windows)
+                self.trim_working_set()
+            except Exception:
+                pass
+
+    # ---------------- Public high-level cleanup APIs -----------------
+    def release_all_llms(self):
+        """Public wrapper to release all tracked LLM instances (non-emergency manual call)."""
+        self._emergency_release_llm_instances()
+
+    def trim_working_set(self):
+        """Attempt to return unused pages to OS (best-effort, platform-specific)."""
+        try:
+            import platform, ctypes
+            if platform.system().lower() == 'windows':
+                PROCESS_SET_QUOTA = 0x0100
+                PROCESS_QUERY_INFORMATION = 0x0400
+                kernel32 = ctypes.windll.kernel32
+                psapi = ctypes.windll.psapi
+                GetCurrentProcess = kernel32.GetCurrentProcess
+                hProc = GetCurrentProcess()
+                psapi.EmptyWorkingSet(hProc)
+            else:
+                # For *nix, reading /proc/self might encourage trimming after gc
+                pass
+        except Exception as e:
+            print(f"Working set trim not supported/failed: {e}")
+
+    def perform_full_cleanup(self) -> Dict[str, float]:
+        """Comprehensive cleanup: release LLMs, GC, trim working set, return new stats."""
+        before = self.get_memory_stats()
+        self.release_all_llms()
+        self.force_collect_garbage()
+        self.trim_working_set()
+        after = self.get_memory_stats()
+        delta = after['available_mb'] - before['available_mb']
+        print(f"Post-query full cleanup reclaimed {delta:.2f}MB (avail: {after['available_mb']:.2f}MB)")
+        return after
+
+    # --------------- Critical section helpers ---------------
+    @contextmanager
+    def suspend_emergency(self, reason: str = ""):
+        """Context manager to temporarily suspend emergency LLM releases.
+
+        Use around critical sections (e.g., active LLM generation) to prevent
+        the monitor thread from tearing down the only LLM mid-call, which can
+        cause intermittent generation failures.
+        """
+        with self._lock:
+            self._suspend_emergency_depth += 1
+            if reason:
+                print(f"⏸️ Suspended emergency releases (depth={self._suspend_emergency_depth}) reason={reason}")
+        try:
+            yield
+        finally:
+            with self._lock:
+                self._suspend_emergency_depth = max(0, self._suspend_emergency_depth - 1)
+                if reason:
+                    print(f"▶️ Resumed emergency releases (depth={self._suspend_emergency_depth}) reason={reason}")
+    
+    def start_monitor(self):
+        """Start the background memory monitor."""
+        with self._lock:
+            if self._monitor_thread is None or not self._monitor_thread.is_alive():
+                self._keep_monitoring = True
+                self._monitor_thread = threading.Thread(
+                    target=self._monitor_memory,
+                    daemon=True,
+                    name="MemoryMonitorThread"
+                )
+                self._monitor_thread.start()
+                print("Memory monitor started in background")
+    
+    def stop_monitor(self):
+        """Stop the background memory monitor."""
+        with self._lock:
+            self._keep_monitoring = False
+            if self._monitor_thread and self._monitor_thread.is_alive():
+                self._monitor_thread.join(timeout=1.0)
+                print("Memory monitor stopped")
+
+# Create a global memory manager instance
+memory_manager = MemoryManager(
+    threshold_mb=500.0,   # 500MB — warning/action threshold (reduced from 1000MB)
+    critical_mb=300.0,    # 300MB — BSOD-protection hard floor (unchanged)
+    check_interval=1.0,   # Check every second
+    monitor_active=True   # Start monitoring immediately
+)
+
+class TimeoutManager:
+    """Manages operation timeouts with proper cleanup."""
+    
+    @staticmethod
+    def run_with_timeout(func: Callable, timeout: float, *args, **kwargs) -> Any:
+        """
+        Run a function with a timeout.
+        
+        Args:
+            func: Function to run
+            timeout: Timeout in seconds
+            *args, **kwargs: Arguments to pass to the function
+            
+        Returns:
+            The result of the function
+            
+        Raises:
+            TimeoutError: If the function times out
+        """
+        result = None
+        exception = None
+        completed = threading.Event()
+        
+        # Thread function
+        def worker():
+            nonlocal result, exception
+            try:
+                result = func(*args, **kwargs)
+            except Exception as e:
+                exception = e
+            finally:
+                completed.set()
+        
+        # Start the worker thread
+        thread = threading.Thread(target=worker, daemon=True)
+        thread.start()
+        
+        # Wait for completion or timeout
+        if not completed.wait(timeout):
+            # Force garbage collection before raising timeout
+            memory_manager.force_collect_garbage()
+            
+            # Raise timeout error
+            raise TimeoutError(f"Operation timed out after {timeout} seconds")
+        
+        # If there was an exception, raise it
+        if exception is not None:
+            raise exception
+        
+        return result
+
+def safe_check_memory(threshold_mb: Optional[float] = None) -> bool:
+    """
+    Safely check if there's enough memory available.
+    Also performs garbage collection if memory is low.
+    
+    Args:
+        threshold_mb: Optional threshold in MB (uses memory manager's threshold if None)
+        
+    Returns:
+        True if memory is safe, False if it's below threshold
+    """
+    # Use memory manager's threshold if none provided
+    if threshold_mb is None:
+        threshold_mb = memory_manager.threshold_mb
+    
+    # Get current memory stats
+    stats = memory_manager.get_memory_stats()
+    
+    # Log memory status
+    print(f"Memory status: {stats['available_mb']:.2f}MB available of {stats['total_mb']:.2f}MB total")
+    
+    # If memory is below threshold, run garbage collection
+    if stats['available_mb'] < threshold_mb:
+        memory_manager.force_collect_garbage()
+        
+        # Get updated stats after garbage collection
+        stats = memory_manager.get_memory_stats()
+        
+        # If still below threshold, return False
+        if stats['available_mb'] < threshold_mb:
+            print(f"⚠️ Low memory warning: Only {stats['available_mb']:.2f}MB available, need at least {threshold_mb:.2f}MB")
+            return False
+    
+    return True

Agent/merge_lora_to_ollama.py

@@ -0,0 +1,244 @@
+"""Merge a PEFT LoRA adapter (Gemma 1B) into base weights and prepare an Ollama model directory.
+
+Usage (after installing extra deps in requirements.txt):
+    python Agent/merge_lora_to_ollama.py \
+        --base google/gemma-1.1-1b-it \
+        --adapter-dir Agent/Gemma3_1B_weights \
+        --out-dir merged_gemma1b_text2sql \
+        --ollama-model-name gemma1b-text2sql
+
+Then create the Ollama model:
+    (In PowerShell)
+    ollama create gemma1b-text2sql -f merged_gemma1b_text2sql/Modelfile
+
+Finally set FINETUNED_MODEL_NAME=gemma1b-text2sql in your .env (or environment) and run the agent.
+
+Notes:
+- Ensure you used the SAME base checkpoint for finetuning as you pass via --base.
+- Script keeps everything on CPU (device_map="cpu") to stay RAM friendly; you can add --load-in-8bit if you install bitsandbytes and have GPU.
+"""
+import argparse
+import json
+import os
+import sys
+from pathlib import Path
+
+from huggingface_hub import HfApi
+try:  # Compatibility across hub versions
+    from huggingface_hub.errors import RepositoryNotFoundError  # type: ignore
+except Exception:  # Older versions
+    class RepositoryNotFoundError(Exception):
+        pass
+
+import torch
+from transformers import AutoModelForCausalLM, AutoTokenizer
+from peft import PeftModel
+
+
+def _detect_base_from_adapter(adapter_dir: str) -> str | None:
+    p = Path(adapter_dir)
+    resolved = p.resolve()
+    print(f"[auto-base] Inspecting adapter_dir: {adapter_dir} -> {resolved}")
+    if not resolved.exists():
+        print(f"[auto-base] Directory does not exist: {resolved}")
+        # Attempt smart fallbacks (common when user runs inside Agent/ already)
+        candidates = []
+        name = p.name
+        cwd = Path.cwd()
+        candidates.append(cwd / name)
+        candidates.append(cwd.parent / name)
+        candidates.append(cwd.parent / 'Agent' / name)
+        for cand in candidates:
+            if cand.exists():
+                print(f"[auto-base] Found alternative adapter dir: {cand}")
+                resolved = cand
+                break
+        if not resolved.exists():
+            print(f"[auto-base] Fallback search failed. Tried: {[str(c) for c in candidates]}")
+            return None
+    cfg_path = resolved / "adapter_config.json"
+    if not cfg_path.exists():
+        try:
+            contents = [c.name for c in resolved.iterdir()]
+        except Exception:
+            contents = []
+        print(f"[auto-base] adapter_config.json not found in {resolved}. Contents: {contents}")
+        return None
+    try:
+        raw = cfg_path.read_text(encoding="utf-8")
+        print(f"[auto-base] Found adapter_config.json ({len(raw)} bytes)")
+        data = json.loads(raw)
+        base = data.get("base_model_name_or_path")
+        print(f"[auto-base] base_model_name_or_path: {base}")
+        return base
+    except Exception as e:
+        print(f"[auto-base] Failed reading adapter_config.json: {e}")
+        return None
+
+
+def merge_adapter(base: str, adapter_dir: str, out_dir: str, ollama_model_name: str, template_temperature: float = 0.05, hf_token: str | None = None, local_files_only: bool = False, auto_base: bool = False, trust_remote_code: bool = True):
+    out_path = Path(out_dir)
+    out_path.mkdir(parents=True, exist_ok=True)
+
+    if auto_base or base.lower() == "auto":
+        detected = _detect_base_from_adapter(adapter_dir)
+        if detected:
+            print(f"[auto-base] Detected base model from adapter_config: {detected}")
+            base = detected
+        else:
+            raise RuntimeError("Could not auto-detect base model; supply --base explicitly.")
+
+    auth_kwargs = {}
+    if hf_token:
+        # Only pass 'token' (use_auth_token deprecated & causes conflict if both set)
+        auth_kwargs = {"token": hf_token}
+        os.environ.setdefault("HF_TOKEN", hf_token)
+        os.environ.setdefault("HUGGINGFACEHUB_API_TOKEN", hf_token)
+        print(f"Using HF token (masked): {hf_token[:8]}... (length {len(hf_token)})")
+
+    # Quick repo existence check (skipped if local_files_only)
+    if not local_files_only:
+        try:
+            HfApi().model_info(base, token=hf_token)
+        except RepositoryNotFoundError:
+            print(f"\n[ERROR] Base model repo '{base}' not found on Hugging Face.\n")
+            print("Troubleshooting:")
+            print("  - Verify the exact model id: e.g. 'google/gemma-2-2b-it' or 'google/gemma-1.1-2b-it'.")
+            print("  - If it's a gated/private model ensure your token has accepted the terms.")
+            print("  - Pass --hf-token YOUR_TOKEN or set HF_TOKEN env var.")
+            return 2
+        except Exception as e:
+            print(f"[WARN] Could not verify model existence pre-download: {e}")
+
+    print(f"Loading base model: {base} (local_files_only={local_files_only})")
+    try:
+        tokenizer = AutoTokenizer.from_pretrained(base, local_files_only=local_files_only, **auth_kwargs)
+    except Exception as e:
+        msg = str(e)
+        if "403" in msg or "Forbidden" in msg:
+            print("\n[AUTH ERROR] 403 while fetching tokenizer/config.")
+            print("Checklist:")
+            print("  1. Ensure token has: 'Read access to public gated models'.")
+            print("  2. Visit model page and accept license (if not already).")
+            print("  3. If using a fine-grained token, enable 'Gated repos' permission.")
+            print("  4. Retry with: --hf-token $env:HF_TOKEN (PowerShell: $env:HF_TOKEN='xxxx')")
+            print("  5. If model already cached on another machine, copy that ~/.cache/huggingface/hub directory.")
+        raise
+
+    # Keep on CPU to reduce VRAM needs; if you want faster and have GPU, remove device_map and add torch_dtype.
+    model_kwargs = dict(
+        device_map="cpu",
+        local_files_only=local_files_only,
+        torch_dtype=torch.float16 if torch.cuda.is_available() else torch.float32,
+        **auth_kwargs,
+    )
+    if trust_remote_code:
+        model_kwargs["trust_remote_code"] = True
+    # Lower RAM peak during load
+    model_kwargs["low_cpu_mem_usage"] = True
+
+    try:
+        print("[stage] Downloading / loading base model weights...")
+        base_model = AutoModelForCausalLM.from_pretrained(base, **model_kwargs)
+        print("[stage] Base model loaded successfully.")
+    except ValueError as ve:
+        msg = str(ve)
+        if "does not recognize this architecture" in msg or "model type" in msg:
+            print("[model-load] Unrecognized model type. Attempting retry after upgrading transformers may be required.")
+            print("Suggestion: pip install --upgrade transformers huggingface_hub --no-cache-dir")
+            if not trust_remote_code:
+                print("Retrying with trust_remote_code=True ...")
+                model_kwargs["trust_remote_code"] = True
+                base_model = AutoModelForCausalLM.from_pretrained(base, **model_kwargs)
+            else:
+                # Last resort: advise source install
+                raise
+        else:
+            raise
+    except Exception as e:
+        msg = str(e)
+        if "403" in msg or "Forbidden" in msg:
+            print("\n[AUTH ERROR] 403 while fetching model weights (config or pytorch model). See guidance above.")
+        raise
+
+    print(f"Loading LoRA adapter from: {adapter_dir}")
+    peft_model = PeftModel.from_pretrained(base_model, adapter_dir)
+    print("[stage] LoRA adapter loaded.")
+
+    print("Merging LoRA weights into base weights (this may take a minute)...")
+    merged_model = peft_model.merge_and_unload()
+    print("[stage] Merge complete in-memory.")
+
+    print(f"Saving merged model to: {out_dir}")
+    merged_model.save_pretrained(out_dir)
+    tokenizer.save_pretrained(out_dir)
+    print("[stage] Saved model and tokenizer.")
+
+    # Create a simple Modelfile for Ollama (avoid nested triple-quote confusion)
+    modelfile_lines = [
+        # Use current directory as model context so Modelfile works regardless of parent path
+        "FROM .",
+        f"PARAMETER temperature {template_temperature}",
+        "# Basic template; adjust if you have a custom chat format.",
+        'TEMPLATE """{{ .Prompt }}"""',
+        ""
+    ]
+    modelfile = "\n".join(modelfile_lines)
+    (out_path / "Modelfile").write_text(modelfile, encoding="utf-8")
+
+    meta = {
+        "base": base,
+        "adapter_dir": adapter_dir,
+        "merged_dir": out_dir,
+        "ollama_model_name": ollama_model_name,
+        "temperature": template_temperature,
+    }
+    (out_path / "merge_metadata.json").write_text(json.dumps(meta, indent=2), encoding="utf-8")
+
+    print("Merge complete. Next steps:")
+    print(f"  1. ollama create {ollama_model_name} -f {out_dir}/Modelfile")
+    print(f"  2. Set FINETUNED_MODEL_NAME={ollama_model_name}")
+    print("  3. Run your agent code again.")
+
+
+def main():
+    if len(sys.argv) == 1:
+        print("\n[merge_lora_to_ollama] Missing arguments. Example usage:\n")
+        print("python Agent/merge_lora_to_ollama.py \\")
+        print("  --base google/gemma-1.1-1b-it \\")
+        print("  --adapter-dir Agent/Gemma3_1B_weights \\")
+        print("  --out-dir merged_gemma1b_text2sql \\")
+        print("  --ollama-model-name gemma1b-text2sql\n")
+        return 1
+
+    parser = argparse.ArgumentParser(description="Merge a Gemma LoRA adapter for Ollama usage.")
+    parser.add_argument("--base", required=True, help="Base HF model id used during finetuning (e.g., google/gemma-3-1b-it or 'auto' to read adapter_config)")
+    parser.add_argument("--auto-base", action="store_true", help="Ignore --base value and read base from adapter_config.json")
+    parser.add_argument("--adapter-dir", required=True, help="Path to the LoRA adapter directory (with adapter_model.safetensors)")
+    parser.add_argument("--out-dir", required=True, help="Directory to write merged weights and Modelfile")
+    parser.add_argument("--ollama-model-name", required=True, help="Target Ollama model name to create")
+    parser.add_argument("--temperature", type=float, default=0.05, help="Default temperature for Modelfile")
+    parser.add_argument("--hf-token", help="Hugging Face access token (optional; else read HF_TOKEN / HUGGINGFACEHUB_API_TOKEN env)")
+    parser.add_argument("--local-files-only", action="store_true", help="Do not attempt to download; require weights to be cached locally")
+    parser.add_argument("--no-trust-remote-code", action="store_true", help="Disable trust_remote_code (not recommended for Gemma 3)")
+    args = parser.parse_args()
+
+    hf_token = args.hf_token or os.environ.get("HF_TOKEN") or os.environ.get("HUGGINGFACEHUB_API_TOKEN")
+    if not hf_token:
+        print("[INFO] No HF token provided. If the model is gated/private, pass --hf-token or set HF_TOKEN env.")
+
+    merge_adapter(
+        base=args.base,
+        adapter_dir=args.adapter_dir,
+        out_dir=args.out_dir,
+        ollama_model_name=args.ollama_model_name,
+        template_temperature=args.temperature,
+        hf_token=hf_token,
+        local_files_only=args.local_files_only,
+        auto_base=args.auto_base,
+        trust_remote_code=not args.no_trust_remote_code,
+    )
+
+
+if __name__ == "__main__":
+    main()

Agent/merge_qwen_to_ollama.py

@@ -0,0 +1,181 @@
+
+"""Merge a PEFT LoRA adapter (Qwen 2.5 3b) into base weights and prepare an Ollama model directory.
+
+This is a standalone, non-destructive helper. It follows the same pattern as
+`Agent/merge_lora_to_ollama.py` but defaults to a Qwen base and includes flags
+for optional low-memory / 8-bit loading.
+
+Usage (PowerShell):
+   Agent/merge_qwen_to_ollama.py \
+    --base Qwenpython/Qwen2.5-3B-Instruct \
+    --adapter-dir Agent/Qwen2.5_3b_weights \
+    --out-dir merged_qwen2_5_3b_text2sql \
+    --ollama-model-name qwen2.5-text2sql
+
+Notes:
+- Keeps merge on CPU to minimize peak memory.
+- If you have a GPU and want faster tokenizers/model ops, see OLLAMA GPU notes file.
+"""
+import argparse
+import json
+import os
+import sys
+from pathlib import Path
+
+try:
+    from huggingface_hub import HfApi
+except Exception:
+    HfApi = None
+
+import torch
+from transformers import AutoModelForCausalLM, AutoTokenizer
+from peft import PeftModel
+from transformers import BitsAndBytesConfig
+
+def _detect_base_from_adapter(adapter_dir: str) -> str | None:
+    p = Path(adapter_dir)
+    resolved = p.resolve()
+    print(f"[auto-base] Inspecting adapter_dir: {adapter_dir} -> {resolved}")
+    if not resolved.exists():
+        print(f"[auto-base] Directory does not exist: {resolved}")
+        # Attempt smart fallbacks (common when user runs inside Agent/ already)
+        candidates = []
+        name = p.name
+        cwd = Path.cwd()
+        candidates.append(cwd / name)
+        candidates.append(cwd.parent / name)
+        candidates.append(cwd.parent / 'Agent' / name)
+        for cand in candidates:
+            if cand.exists():
+                print(f"[auto-base] Found alternative adapter dir: {cand}")
+                resolved = cand
+                break
+        if not resolved.exists():
+            print(f"[auto-base] Fallback search failed. Tried: {[str(c) for c in candidates]}")
+            return None
+    cfg_path = resolved / "adapter_config.json"
+    if not cfg_path.exists():
+        try:
+            contents = [c.name for c in resolved.iterdir()]
+        except Exception:
+            contents = []
+        print(f"[auto-base] adapter_config.json not found in {resolved}. Contents: {contents}")
+        return None
+    try:
+        raw = cfg_path.read_text(encoding="utf-8")
+        print(f"[auto-base] Found adapter_config.json ({len(raw)} bytes)")
+        data = json.loads(raw)
+        base = data.get("base_model_name_or_path")
+        print(f"[auto-base] base_model_name_or_path: {base}")
+        return base
+    except Exception as e:
+        print(f"[auto-base] Failed reading adapter_config.json: {e}")
+        return None
+
+def merge_adapter(base: str, adapter_dir: str, out_dir: str, ollama_model_name: str, template_temperature: float = 0.05, hf_token: str | None = None, local_files_only: bool = False, trust_remote_code: bool = True, auto_base: bool = False):
+    out_path = Path(out_dir)
+    out_path.mkdir(parents=True, exist_ok=True)
+
+    # If requested, detect base from adapter_config.json
+    if auto_base or (isinstance(base, str) and base.lower() == "auto"):
+        detected = _detect_base_from_adapter(adapter_dir)
+        if detected:
+            print(f"[auto-base] Detected base model from adapter_config: {detected}")
+            base = detected
+        else:
+            raise RuntimeError("Could not auto-detect base model; supply --base explicitly.")
+
+    # Load the tokenizer first, it's small and fast
+    try:
+        tokenizer = AutoTokenizer.from_pretrained(base, local_files_only=local_files_only)
+    except Exception as e:
+        print(f"Failed loading tokenizer for base '{base}': {e}")
+        raise
+
+    # Configure 4-bit quantization with the necessary offload flag
+    quantization_config = BitsAndBytesConfig(
+        load_in_4bit=True,
+        bnb_4bit_quant_type="nf4",
+        bnb_4bit_compute_dtype=torch.float16,
+        bnb_4bit_use_double_quant=True,
+        llm_int8_enable_fp32_cpu_offload=True, # THIS IS CRUCIAL
+    )
+
+    print(f"Loading base model: {base} (using 4-bit quantization with auto device mapping) 🧠💡")
+
+    try:
+        # Use the BitsAndBytesConfig and let device_map="auto" handle the offloading
+        base_model = AutoModelForCausalLM.from_pretrained(
+            base,
+            quantization_config=quantization_config,
+            device_map="auto",
+            low_cpu_mem_usage=True,
+            local_files_only=local_files_only,
+            trust_remote_code=trust_remote_code,
+        )
+    except Exception as e:
+        print(f"Failed loading base model '{base}': {e}")
+        raise
+
+    print(f"Loading LoRA adapter from: {adapter_dir}")
+    peft_model = PeftModel.from_pretrained(base_model, adapter_dir)
+    print("LoRA adapter loaded.")
+
+    print("Merging LoRA weights into base weights (in-memory)...")
+    merged_model = peft_model.merge_and_unload()
+    print("Merge complete.")
+
+    print(f"Saving merged model to: {out_dir}")
+    merged_model.save_pretrained(out_dir)
+    tokenizer.save_pretrained(out_dir)
+
+    modelfile_lines = [
+        "FROM .",
+        f"PARAMETER temperature {template_temperature}",
+        'TEMPLATE """{{ .Prompt }}"""',
+        "",
+    ]
+    (out_path / "Modelfile").write_text("\n".join(modelfile_lines), encoding="utf-8")
+
+    meta = {
+        "base": base,
+        "adapter_dir": adapter_dir,
+        "merged_dir": out_dir,
+        "ollama_model_name": ollama_model_name,
+        "temperature": template_temperature,
+    }
+    (out_path / "merge_metadata.json").write_text(json.dumps(meta, indent=2), encoding="utf-8")
+
+    print("Merge complete. Next steps:")
+    print(f" 1. ollama create {ollama_model_name} -f {out_dir}/Modelfile")
+    print(f" 2. Set FINETUNED_MODEL_NAME={ollama_model_name} in your environment")
+    print(" 3. Run your agent code again.")
+
+def main():
+    parser = argparse.ArgumentParser(description="Merge a Qwen LoRA adapter for Ollama usage.")
+    parser.add_argument("--base", required=True, help="Base HF model id used during finetuning (default: qwen2.5:3b-instruct)")
+    parser.add_argument("--auto-base", action="store_true", help="Ignore --base value and read base from adapter_config.json")
+    parser.add_argument("--adapter-dir", required=True, help="Path to the LoRA adapter directory (with adapter_model.safetensors)")
+    parser.add_argument("--out-dir", required=True, help="Directory to write merged weights and Modelfile")
+    parser.add_argument("--ollama-model-name", required=True, help="Target Ollama model name to create")
+    parser.add_argument("--temperature", type=float, default=0.05, help="Default temperature for Modelfile")
+    parser.add_argument("--hf-token", help="Hugging Face access token (optional)")
+    parser.add_argument("--local-files-only", action="store_true", help="Do not attempt to download; require weights to be cached locally")
+    parser.add_argument("--no-trust-remote-code", action="store_true", help="Disable trust_remote_code (not recommended for some models)")
+    args = parser.parse_args()
+
+    hf_token = args.hf_token or os.environ.get("HF_TOKEN") or os.environ.get("HUGGINGFACEHUB_API_TOKEN")
+
+    merge_adapter(
+        base=args.base,
+        adapter_dir=args.adapter_dir,
+        out_dir=args.out_dir,
+        ollama_model_name=args.ollama_model_name,
+        template_temperature=args.temperature,
+        hf_token=hf_token,
+        local_files_only=args.local_files_only,
+        trust_remote_code=not args.no_trust_remote_code,
+    )
+
+if __name__ == "__main__":
+    main()

Agent/merged_model_test.py

@@ -0,0 +1,26 @@
+import json
+from safetensors.numpy import load_file
+from pathlib import Path
+
+base = Path(r"e:\Capstone_Project\text2sql-agent-mcp\merged_qwen2_5_3b_text2sql")
+safetensor = base / "model.safetensors"
+cfg_path = base / "config.json"
+
+cfg = json.loads(cfg_path.read_text(encoding="utf-8"))
+hidden = cfg.get("hidden_size")
+print("expected attn_q shape (hidden, hidden) =", hidden, hidden)
+
+print("Loading safetensors (this may use RAM)...")
+tensors = load_file(str(safetensor))
+
+# find candidate keys
+candidates = [k for k in tensors.keys() if "attn_q.weight" in k or "blk.0.attn_q.weight" in k]
+if not candidates:
+    # also show some top-level keys to help debug
+    print("No attn_q keys found. Showing first 40 keys in the safetensor:")
+    for k in list(tensors.keys())[:40]:
+        print("  ", k)
+else:
+    print("Found attn_q keys (showing up to 30):")
+    for k in candidates[:30]:
+        print(k, "->", tensors[k].shape, "dtype:", tensors[k].dtype)

Agent/migrate_to_neon.py

@@ -0,0 +1,54 @@
+import os
+import asyncio
+from sqlalchemy import create_engine, text
+from sqlalchemy.engine import URL
+import pandas as pd
+from dotenv import load_dotenv
+
+# Try to find .env in current dir and Agent dir
+load_dotenv()
+load_dotenv(dotenv_path=os.path.join(os.path.dirname(__file__), '.env'))
+load_dotenv(dotenv_path=os.path.join(os.path.dirname(__file__), '..', '.env'))
+
+# LOCAL_DB = "postgresql://postgres:postgres@localhost:5432/employee"
+LOCAL_DB = os.environ.get("DB_CONNECTION_URL")
+# NEON_DB = Your new Neon connection string
+NEON_DB = os.environ.get("NEON_DB_URL")
+
+async def migrate():
+    if not NEON_DB:
+        print("❌ Error: NEON_DB_URL not found in environment variables.")
+        return
+
+    print("🚀 Starting Cloud Migration to Neon.tech...")
+    
+    # 1. Connect to Local
+    local_engine = create_engine(LOCAL_DB)
+    
+    # 2. Connect to Neon (Ensure SSL)
+    neon_uri = NEON_DB
+    if 'sslmode=' not in neon_uri:
+        sep = '&' if '?' in neon_uri else '?'
+        neon_uri += f"{sep}sslmode=require"
+    
+    neon_engine = create_engine(neon_uri)
+
+    tables = ["department", "employee", "dept_emp", "dept_manager", "salary", "title"]
+
+    for table in tables:
+        print(f"📦 Migrating table: {table}...")
+        try:
+            # Read data from local
+            df = pd.read_sql_table(table, local_engine)
+            print(f"   - Found {len(df)} rows.")
+            
+            # Write to Neon (replace if exists)
+            df.to_sql(table, neon_engine, if_exists='replace', index=False)
+            print(f"   ✅ {table} successfully uploaded to Neon.")
+        except Exception as e:
+            print(f"   ❌ Failed to migrate {table}: {e}")
+
+    print("\n🎉 Migration Complete! Your cloud database is ready.")
+
+if __name__ == "__main__":
+    asyncio.run(migrate())

Agent/schema_profile.py

@@ -0,0 +1,285 @@
+import os
+import json
+import time
+import uuid
+import threading
+from typing import Any, Dict, List, Optional
+
+
+class SchemaProfile:
+    """
+    Runtime schema profile representing the active database.
+    Stores tables, columns, PK/FK metadata, row counts, sample rows, and
+    a digest string suitable for injecting into LLM prompts.
+    """
+
+    SAMPLE_ROW_LIMIT = 3
+    MAX_DIGEST_TABLES = 40
+    MAX_DIGEST_COLUMNS = 25
+
+    def __init__(self, name: str, db_url: Optional[str] = None, is_demo: bool = False):
+        self.name = name
+        self.db_url = db_url
+        self.is_demo = is_demo
+        self.tables: List[str] = []
+        self.columns_by_table: Dict[str, List[Dict[str, Any]]] = {}
+        self.pk_by_table: Dict[str, List[str]] = {}
+        self.fk_by_table: Dict[str, List[Dict[str, Any]]] = {}
+        self.row_counts: Dict[str, int] = {}
+        self.sample_rows: Dict[str, List[Dict[str, Any]]] = {}
+        self.available_tables_set: set = set()
+
+    @property
+    def is_employee_db(self) -> bool:
+        # Treat as employee demo if explicitly marked or name matches
+        if self.is_demo:
+            return True
+        n = (self.name or "").lower()
+        return "employee" in n or n == "demo_employee"
+
+    # ------------------------------------------------------------------
+    # Population
+    # ------------------------------------------------------------------
+    def populate_from_inspector(self, inspector: Any, engine: Optional[Any] = None) -> None:
+        """Fill all metadata from a SQLAlchemy inspector (and optionally sample
+        row counts / preview rows via a sync engine connection)."""
+        try:
+            tables = list(
+                t for t in (inspector.get_table_names() + inspector.get_view_names())
+                if not t.startswith("_")
+            )
+        except Exception as e:
+            print(f"[SCHEMA-PROFILE] inspector table listing failed: {e}")
+            tables = []
+
+        self.tables = sorted(tables)
+        self.available_tables_set = set(self.tables)
+        self.columns_by_table = {}
+        self.pk_by_table = {}
+        self.fk_by_table = {}
+
+        for t in self.tables:
+            try:
+                self.columns_by_table[t] = inspector.get_columns(t)
+            except Exception:
+                self.columns_by_table[t] = []
+            try:
+                pks = inspector.get_pk_constraint(t)
+                self.pk_by_table[t] = (pks or {}).get("constrained_columns", []) or []
+            except Exception:
+                self.pk_by_table[t] = []
+            try:
+                self.fk_by_table[t] = inspector.get_foreign_keys(t) or []
+            except Exception:
+                self.fk_by_table[t] = []
+
+        if engine is not None:
+            self._collect_row_stats(engine)
+
+    def _collect_row_stats(self, engine: Any) -> None:
+        """Best-effort row counts + small sample rows using a sync engine."""
+        from sqlalchemy import text as sql_text
+
+        self.row_counts = {}
+        self.sample_rows = {}
+        try:
+            with engine.connect() as conn:
+                for t in self.tables:
+                    # Row count (quoted identifier first, unquoted fallback)
+                    try:
+                        cnt = conn.execute(sql_text(f'SELECT COUNT(*) FROM "{t}"')).scalar()
+                        self.row_counts[t] = int(cnt) if cnt is not None else 0
+                    except Exception:
+                        try:
+                            cnt = conn.execute(sql_text(f"SELECT COUNT(*) FROM {t}")).scalar()
+                            self.row_counts[t] = int(cnt) if cnt is not None else 0
+                        except Exception:
+                            self.row_counts[t] = 0
+                    # Sample rows
+                    if self.row_counts.get(t, 0) > 0:
+                        try:
+                            result = conn.execute(
+                                sql_text(f'SELECT * FROM "{t}" LIMIT {self.SAMPLE_ROW_LIMIT}')
+                            )
+                            rows = result.mappings().all()
+                            self.sample_rows[t] = [
+                                {k: _json_safe(v) for k, v in dict(r).items()} for r in rows
+                            ]
+                        except Exception:
+                            try:
+                                result = conn.execute(
+                                    sql_text(f"SELECT * FROM {t} LIMIT {self.SAMPLE_ROW_LIMIT}")
+                                )
+                                rows = result.mappings().all()
+                                self.sample_rows[t] = [
+                                    {k: _json_safe(v) for k, v in dict(r).items()} for r in rows
+                                ]
+                            except Exception:
+                                self.sample_rows[t] = []
+                    else:
+                        self.sample_rows[t] = []
+        except Exception as e:
+            print(f"[SCHEMA-PROFILE] row stats collection failed: {e}")
+
+    # ------------------------------------------------------------------
+    # Digest for LLM prompts
+    # ------------------------------------------------------------------
+    def build_digest(self, max_tables: Optional[int] = None) -> str:
+        max_tables = max_tables or self.MAX_DIGEST_TABLES
+        lines: List[str] = []
+        lines.append(f"Database schema name: {self.name}")
+        lines.append(f"Number of tables/views: {len(self.tables)}")
+        lines.append("Tables and columns:")
+        for t in self.tables[:max_tables]:
+            cols = self.columns_by_table.get(t, [])
+            col_parts = []
+            for c in cols[: self.MAX_DIGEST_COLUMNS]:
+                ctype = str(c.get("type", ""))[:24]
+                col_parts.append(f"{c.get('name')} {ctype}".strip())
+            colstr = ", ".join(col_parts) if col_parts else "(no columns)"
+            rc = self.row_counts.get(t)
+            lines.append(f"  - {t} ({colstr})  [rows={rc if rc is not None else '?'}]")
+            fks = self.fk_by_table.get(t, [])
+            for fk in fks[:4]:
+                ref = fk.get("referred_table")
+                lcols = fk.get("constrained_columns") or []
+                rcols = fk.get("referred_columns") or []
+                if ref:
+                    lines.append(f"      FK: {lcols} -> {ref}({rcols})")
+            if len(cols) > self.MAX_DIGEST_COLUMNS:
+                lines.append(f"      ... {len(cols) - self.MAX_DIGEST_COLUMNS} more columns")
+        if len(self.tables) > max_tables:
+            lines.append(f"  ... {len(self.tables) - max_tables} more tables")
+        return "\n".join(lines)
+
+    def summarize(self) -> Dict[str, Any]:
+        """Compact JSON-friendly summary (for API responses / UI)."""
+        return {
+            "name": self.name,
+            "db_url": self.db_url,
+            "is_demo": self.is_demo,
+            "is_employee_db": self.is_employee_db,
+            "tables": [
+                {
+                    "name": t,
+                    "row_count": self.row_counts.get(t),
+                    "columns": [c.get("name") for c in self.columns_by_table.get(t, [])],
+                    "sample_rows": self.sample_rows.get(t, []),
+                }
+                for t in self.tables
+            ],
+        }
+
+
+def _json_safe(v: Any) -> Any:
+    try:
+        json.dumps(v)
+        return v
+    except (TypeError, ValueError):
+        return str(v)
+
+
+class ProfileRegistry:
+    """
+    Registry of user-registered database profiles.
+
+    Each registry entry stores lightweight metadata (id, name, db_url, kind).
+    Full introspection happens lazily when a profile is switched to (the
+    agent's initialize_database walks the schema at that moment).
+    """
+
+    def __init__(self, store_path: str):
+        self.store_path = store_path
+        self._lock = threading.Lock()
+        self.profiles: Dict[str, Dict[str, Any]] = {}
+        self.active_id: Optional[str] = None
+        self.load()
+
+    # ------------------------------------------------------------------
+    def load(self) -> None:
+        try:
+            if os.path.exists(self.store_path):
+                with open(self.store_path, "r", encoding="utf-8") as fh:
+                    data = json.load(fh)
+                self.profiles = data.get("profiles", {})
+                self.active_id = data.get("active_id")
+        except Exception as e:
+            print(f"[PROFILE-REGISTRY] load failed: {e}")
+            self.profiles = {}
+            self.active_id = None
+
+    def save(self) -> None:
+        with self._lock:
+            try:
+                os.makedirs(os.path.dirname(self.store_path), exist_ok=True)
+                with open(self.store_path, "w", encoding="utf-8") as fh:
+                    json.dump(
+                        {"profiles": self.profiles, "active_id": self.active_id},
+                        fh,
+                        indent=2,
+                    )
+            except Exception as e:
+                print(f"[PROFILE-REGISTRY] save failed: {e}")
+
+    # ------------------------------------------------------------------
+    def list(self) -> List[Dict[str, Any]]:
+        out = []
+        for pid, p in self.profiles.items():
+            out.append(
+                {
+                    "id": pid,
+                    "name": p.get("name"),
+                    "kind": p.get("kind", "user"),
+                    "db_url": p.get("db_url"),
+                    "created_at": p.get("created_at"),
+                    "active": pid == self.active_id,
+                }
+            )
+        return sorted(out, key=lambda p: (not p["active"], p.get("created_at") or 0))
+
+    def get(self, profile_id: str) -> Optional[Dict[str, Any]]:
+        return self.profiles.get(profile_id)
+
+    def active(self) -> Optional[Dict[str, Any]]:
+        if self.active_id and self.active_id in self.profiles:
+            return {"id": self.active_id, **self.profiles[self.active_id]}
+        return None
+
+    def add(
+        self,
+        name: str,
+        db_url: str,
+        kind: str = "user",
+        activate: bool = True,
+    ) -> Dict[str, Any]:
+        pid = uuid.uuid4().hex[:12]
+        entry = {
+            "name": name,
+            "db_url": db_url,
+            "kind": kind,
+            "created_at": time.time(),
+        }
+        with self._lock:
+            self.profiles[pid] = entry
+            if activate or self.active_id is None:
+                self.active_id = pid
+        self.save()
+        return {"id": pid, **entry}
+
+    def remove(self, profile_id: str) -> bool:
+        with self._lock:
+            if profile_id not in self.profiles:
+                return False
+            del self.profiles[profile_id]
+            if self.active_id == profile_id:
+                self.active_id = next(iter(self.profiles), None)
+        self.save()
+        return True
+
+    def switch(self, profile_id: str) -> Optional[Dict[str, Any]]:
+        with self._lock:
+            if profile_id not in self.profiles:
+                return None
+            self.active_id = profile_id
+        self.save()
+        return {"id": profile_id, **self.profiles[profile_id]}

Agent/test_adapter_weights.py

@@ -0,0 +1,45 @@
+
+# Set environment variables BEFORE importing torch
+import os
+os.environ["TORCHINDUCTOR_DISABLE"] = "1"
+os.environ["TORCH_COMPILE_DISABLE"] = "1"
+# Optional: force a specific GPU (uncomment if needed)
+# os.environ["CUDA_VISIBLE_DEVICES"] = "0"
+
+from transformers import AutoModelForCausalLM, AutoTokenizer
+from peft import PeftModel
+import torch
+
+# In-code disables for torch.compile and inductor/triton
+torch._dynamo.config.suppress_errors = True
+#torch._dynamo.config.enable = False
+torch.set_float32_matmul_precision('high')
+
+#base = "google/gemma-3-1b-it"
+#adapter = "Agent/Gemma3_1B_weights"
+base = "Qwen/Qwen2.5-3B-Instruct"
+adapter = "Agent/Qwen2.5_3b_weights"
+model = AutoModelForCausalLM.from_pretrained(
+    base,
+    dtype="auto",
+    device_map=None,
+    load_in_8bit=True,
+    low_cpu_mem_usage=True,
+    local_files_only=False  # Set to False to force download if needed
+    #trust_remote_code=True
+)
+model = PeftModel.from_pretrained(model, adapter, local_files_only=True)
+model = model.to("cuda")  # Move model to GPU
+tokenizer = AutoTokenizer.from_pretrained(base)
+
+prompt = "Schema: employee(emp_no, birth_date, first_name, last_name, gender, hire_date),\n" \
+         "department(dept_no, dept_name),\n" \
+         "dept_emp(emp_no, dept_no, from_date, to_date),\n" \
+         "salary(emp_no, salary, from_date, to_date),\n" \
+         "Write PostgreSQL query for highest and lowest salary across all departments. Do not include any explanation."
+inputs = tokenizer(prompt, return_tensors="pt")
+# Move all input tensors to CUDA
+inputs = {k: v.to("cuda") for k, v in inputs.items()}
+
+output = model.generate(**inputs, max_new_tokens=1028)
+print(tokenizer.decode(output[0], skip_special_tokens=True))

Agent/test_merged_model.py

@@ -0,0 +1,19 @@
+from safetensors.numpy import load_file
+from pathlib import Path
+
+base = Path(r"e:\Capstone_Project\text2sql-agent-mcp\merged_qwen2_5_3b_text2sql")
+safetensor = base / "model.safetensors"
+
+print("Loading safetensors (may use RAM)...")
+d = load_file(str(safetensor))
+
+weights = [k for k in d.keys() if k.endswith(".weight")]
+print("weight keys count:", len(weights))
+
+if weights:
+    for k in weights[:200]:
+        print(k, d[k].shape, d[k].dtype)
+else:
+    print("\nNo '.weight' keys found — showing first 120 keys:")
+    for k in list(d.keys())[:120]:
+        print(k)

Agent/test_query_runner.py

@@ -0,0 +1,20 @@
+import asyncio
+import os
+import sys
+# Ensure Agent package path is importable
+sys.path.insert(0, r"e:/Capstone_Project/text2sql-agent-mcp/Agent")
+import backups.agentic_workflow4 as aw
+
+async def run_one():
+    # Initialize DB (use the module's configured DB_CONNECTION_URL when available)
+    db_uri = os.environ.get('DB_CONNECTION_URL') or getattr(aw, 'DB_CONNECTION_URL', None)
+    await aw.initialize_database(db_uri)
+    # Run a single question
+    # Prefix with 're-execute' to force regeneration and bypass any cached error in memory
+    res = await aw.agentic_query_process('re-execute Who are the top 10 highest-paid employees?')
+    print('\n--- AGENT OUTPUT START ---')
+    print(res)
+    print('--- AGENT OUTPUT END ---')
+
+if __name__ == '__main__':
+    asyncio.run(run_one())

Agent/tools/document_scanner.py

@@ -0,0 +1,223 @@
+import os
+try:
+    # Ensure local .env values (OLLAMA_*) are loaded when this module is imported
+    from dotenv import load_dotenv
+    load_dotenv()
+except Exception:
+    pass
+import uuid
+import re
+import base64
+from typing import Dict, Any, Optional
+
+def _mock_result(received: Optional[Dict[str, Any]] = None) -> Dict[str, Any]:
+    out = {"doc_id": str(uuid.uuid4()), "text": "(mock OCR output)", "metadata": {"pages": 1, "confidence": 0.0}}
+    if received:
+        out["received_args"] = received
+    return out
+
+
+def clean_ocr_text(raw: Optional[str]) -> str:
+    """Lightweight post-processing for OCR output to improve readability.
+
+    - turns literal "\\n" sequences into real newlines
+    - unescapes common unicode/HTML escapes when safe
+    - collapses repeated spaces and blank lines
+    - trims and merges lines that were broken mid-sentence
+    """
+    if not raw:
+        return ""
+    s = raw
+    try:
+        # Convert literal backslash-n to actual newline
+        s = s.replace('\\n', '\n')
+    except Exception:
+        pass
+
+    # Try to interpret common escape sequences (conservative)
+    try:
+        s_decoded = s.encode('utf-8', errors='surrogatepass').decode('unicode_escape')
+        # Accept decoded result if it reduces backslash artifacts
+        if '\\n' not in s_decoded:
+            s = s_decoded
+    except Exception:
+        pass
+
+    # Normalize line endings and whitespace
+    s = s.replace('\r\n', '\n').replace('\r', '\n')
+    s = re.sub(r'[ \t]{2,}', ' ', s)
+    # Collapse 3+ newlines into max 2
+    s = re.sub(r'\n{3,}', '\n\n', s)
+
+    # Trim whitespace on each line
+    lines = [ln.strip() for ln in s.splitlines()]
+
+    # Merge lines that likely were broken mid-sentence: if a line doesn't end with
+    # sentence punctuation and the next line begins with a lowercase letter or digit,
+    # join them with a space.
+    merged = []
+    i = 0
+    while i < len(lines):
+        line = lines[i]
+        if i < len(lines) - 1:
+            nxt = lines[i + 1]
+            if line and nxt and not re.search(r'[\.\!\?\:;]$', line) and re.match(r'^[a-z0-9]', nxt):
+                line = line + ' ' + nxt
+                i += 2
+                # Continue merging subsequent similar lines
+                while i < len(lines) and lines[i] and not re.search(r'[\.\!\?\:;]$', line) and re.match(r'^[a-z0-9]', lines[i]):
+                    line = line + ' ' + lines[i]
+                    i += 1
+                merged.append(line)
+                continue
+        merged.append(line)
+        i += 1
+
+    s = '\n'.join([ln for ln in merged if ln])
+
+    # Remove long runs of underscores or pipes
+    s = re.sub(r'[_\|]{2,}', '', s)
+
+    # Strip control chars except common whitespace
+    s = ''.join(ch for ch in s if ord(ch) >= 9 and ord(ch) != 11 and ord(ch) != 12)
+
+    try:
+        import html
+
+        s = html.unescape(s)
+    except Exception:
+        pass
+
+    # Decode literal unicode escapes like \u0027 -> ' and \xA4 -> currency symbol
+    try:
+        def _u_decode(m):
+            code = m.group(1)
+            try:
+                return chr(int(code, 16))
+            except Exception:
+                return m.group(0)
+
+        s = re.sub(r'\\u([0-9A-Fa-f]{4})', _u_decode, s)
+        s = re.sub(r'\\x([0-9A-Fa-f]{2})', lambda m: chr(int(m.group(1), 16)), s)
+    except Exception:
+        pass
+
+    # Remove stray leading 'n' characters that appear before uppercase words (common OCR/newline artifact)
+    try:
+        s = re.sub(r'(?<=\s|^)[nN](?=[A-Z])', '', s)
+    except Exception:
+        pass
+
+    # Ensure space after commas when missing
+    try:
+        s = re.sub(r',(?=[^\s])', ', ', s)
+    except Exception:
+        pass
+
+    # Remove spaces before punctuation
+    try:
+        s = re.sub(r'\s+([,\.\:\;\%])', r'\1', s)
+    except Exception:
+        pass
+
+    # Attempt to fix common mojibake (double-encoded UTF-8) conservatively.
+    # Only apply if there are visible 'Ã' sequences suggesting encoding issues.
+    try:
+        if 'Ã' in s or 'Â' in s:
+            try:
+                candidate = s.encode('latin-1').decode('utf-8')
+                # Accept candidate if it reduces suspicious sequences
+                if candidate.count('Ã') < s.count('Ã'):
+                    s = candidate
+            except Exception:
+                pass
+    except Exception:
+        pass
+
+    return s.strip()
+
+
+def process_image(image_url: Optional[str] = None, image_bytes: Optional[bytes] = None, options: Optional[Dict[str, Any]] = None) -> Dict[str, Any]:
+    """Process an image using local OCR first (pytesseract), with a light transformers fallback.
+
+    Accepts either `image_url` or `image_bytes` (bytes or base64 string). Returns dict: {doc_id, text, metadata}
+    """
+
+    # Lightweight imports that may be missing in minimal environments
+    try:
+        from PIL import Image
+        from io import BytesIO
+        import requests
+    except Exception:
+        # Missing imaging/network deps -> return mock with echo of args
+        return _mock_result(received={"image_url": image_url, "has_bytes": bool(image_bytes)})
+
+    # Normalize image_bytes if base64 string was passed
+    if isinstance(image_bytes, str):
+        try:
+            image_bytes = base64.b64decode(image_bytes)
+        except Exception:
+            image_bytes = None
+
+    # Acquire PIL image
+    img = None
+    try:
+        if image_bytes:
+            img = Image.open(BytesIO(image_bytes)).convert("RGB")
+        elif image_url:
+            resp = requests.get(image_url, timeout=15)
+            resp.raise_for_status()
+            img = Image.open(BytesIO(resp.content)).convert("RGB")
+        else:
+            return _mock_result(received={"image_url": image_url, "has_bytes": False})
+    except Exception:
+        return _mock_result(received={"image_url": image_url, "has_bytes": bool(image_bytes)})
+
+    # PRIORITY: Try local Tesseract (pytesseract) first — lightweight and deterministic on Windows
+
+    try:
+        import pytesseract
+        # Robustly find tesseract executable: prefer PATH, then common Windows install locations
+        try:
+            import shutil
+            found = shutil.which('tesseract')
+        except Exception:
+            found = None
+        if not found:
+            # common install locations on Windows
+            for candidate in (r"C:\Program Files\Tesseract-OCR\tesseract.exe", r"C:\Program Files (x86)\Tesseract-OCR\tesseract.exe"):
+                try:
+                    if os.path.exists(candidate):
+                        found = candidate
+                        break
+                except Exception:
+                    continue
+        try:
+            if found:
+                pytesseract.pytesseract.tesseract_cmd = found
+                print(f"[document_scanner] using tesseract executable: {found}")
+            else:
+                # leave default; pytesseract will raise a helpful error which we catch below
+                print("[document_scanner] tesseract executable not found in PATH or common locations")
+        except Exception:
+            # ignore assignment errors and let pytesseract raise on use
+            pass
+
+        try:
+            if img is not None:
+                print("[document_scanner] attempting PRIMARY pytesseract OCR")
+                ttxt = pytesseract.image_to_string(img)
+                if ttxt and ttxt.strip():
+                    print(f"[document_scanner] pytesseract returned text (len={len(ttxt.strip())})")
+                    return {"doc_id": str(uuid.uuid4()), "text": clean_ocr_text(ttxt), "metadata": {"pages": 1, "confidence": None}}
+                else:
+                    print("[document_scanner] pytesseract returned no text; falling through to model-based fallbacks")
+        except Exception as e_pyt_first:
+            print(f"[document_scanner] primary pytesseract attempt failed: {e_pyt_first}")
+    except Exception:
+        # pytesseract not installed in environment — continue to available backends
+        pass
+
+    # Last resort: return a safe mock that echoes inputs
+    # (Tesseract failed or is not installed — Q&A will still work via Granite Vision if image is saved)
+    return _mock_result(received={"image_url": image_url, "has_bytes": bool(image_bytes)})

Agent/tools/granite_vision.py

@@ -0,0 +1,121 @@
+"""
+Vision MCP tool - visual Q&A via Groq Vision API.
+
+Cloud path  : image -> Groq Vision -> structured answer
+Local path  : image -> Ollama qwen2-vl -> answer (dev only)
+
+Registered in mcp_server.py as 'granite_vision.qa'.
+"""
+import os
+import base64
+import requests
+from typing import Optional
+
+
+def qa(image_path: str, question: str, model_path: Optional[str] = None) -> dict:
+    """Answer a question about a document image using Groq Vision or local Ollama."""
+
+    print(f"[vision.qa] image={image_path} question={question[:80]}")
+
+    # CLOUD MODE: Groq Vision API
+    if os.environ.get("USE_HF_CLOUD", "0") in ("1", "true", "True"):
+        groq_key = os.environ.get("GROQ_API_KEY")
+        groq_model = os.environ.get("GROQ_VISION_MODEL", "qwen/qwen3.8-27b")
+
+        if groq_key:
+            print(f"[granite_vision.qa] CLOUD MODE: {groq_model} via Groq")
+            try:
+                from groq import Groq
+
+                client = Groq(api_key=groq_key)
+
+                with open(image_path, "rb") as f:
+                    img_b64 = base64.b64encode(f.read()).decode("utf-8")
+
+                ext = os.path.splitext(image_path)[-1].lower().lstrip(".")
+                mime = "image/jpeg" if ext in ("jpg", "jpeg") else f"image/{ext}"
+
+                response = client.chat.completions.create(
+                    model=groq_model,
+                    messages=[
+                        {
+                            "role": "user",
+                            "content": [
+                                {
+                                    "type": "text",
+                                    "text": (
+                                        "You are analyzing a document image. "
+                                        "Look carefully at the layout, tables, numbers, and text. "
+                                        "Answer the following question based only on what you see:\n\n"
+                                        + question
+                                    )
+                                },
+                                {
+                                    "type": "image_url",
+                                    "image_url": {
+                                        "url": f"data:{mime};base64,{img_b64}"
+                                    }
+                                }
+                            ]
+                        }
+                    ],
+                    max_tokens=1024,
+                    temperature=0.1
+                )
+
+                answer = response.choices[0].message.content.strip()
+                print(f"[granite_vision.qa] Groq answer ({len(answer)} chars): {answer[:200]}")
+                return {"answer": answer, "model": f"{groq_model} (Groq Vision)"}
+
+            except Exception as e:
+                print(f"[granite_vision.qa] Groq failed: {e}")
+                return {
+                    "answer": f"Groq Vision error: {e}",
+                    "model": groq_model,
+                    "error": str(e)
+                }
+        else:
+            print("[granite_vision.qa] GROQ_API_KEY not set in HF secrets.")
+            return {
+                "answer": "GROQ_API_KEY is not configured. Please add it in HuggingFace Space secrets.",
+                "model": "none",
+                "error": "missing_groq_key"
+            }
+
+    # LOCAL MODE (dev only): Ollama
+    local_model = os.environ.get("GRANITE_MODEL_PATH", "qwen2-vl:2b")
+    print(f"[granite_vision.qa] LOCAL MODE: sending image to Ollama ({local_model})")
+
+    try:
+        with open(image_path, "rb") as f:
+            img_b64 = base64.b64encode(f.read()).decode("utf-8")
+
+        base_url = os.environ.get("OLLAMA_BASE_URL", "http://localhost:11434").rstrip("/")
+        url = f"{base_url}/api/chat"
+        payload = {
+            "model": local_model,
+            "messages": [
+                {
+                    "role": "user",
+                    "content": f"Carefully analyze this document image. {question}",
+                    "images": [img_b64]
+                }
+            ],
+            "stream": False,
+            "keep_alive": 60,
+            "options": {"num_ctx": 1024, "temperature": 0.1}
+        }
+        r = requests.post(url, json=payload, timeout=300)
+        if r.status_code == 200:
+            answer = r.json().get("message", {}).get("content", "").strip()
+            print(f"[granite_vision.qa] Ollama answer ({len(answer)} chars): {answer[:200]}")
+            return {"answer": answer, "model": f"{local_model} (Ollama)"}
+        else:
+            msg = f"Ollama HTTP {r.status_code}: {r.text}"
+            print(f"[granite_vision.qa] {msg}")
+            return {"answer": msg, "model": f"{local_model} (Ollama)", "error": "http_error"}
+
+    except Exception as e:
+        msg = f"[ERROR] Ollama Q&A failed: {e}"
+        print(f"[granite_vision.qa] {msg}")
+        return {"answer": msg, "model": f"{local_model} (Ollama)", "error": str(e)}

Agent/tools/groq_vision_health.py

@@ -0,0 +1,33 @@
+import os
+import json
+import base64
+from typing import Optional
+
+
+def health_check() -> dict:
+    """
+    Quick health check for Groq Vision model.
+    Returns {"ok": bool, "model": str, "error": Optional[str]}
+    """
+    if os.environ.get("USE_HF_CLOUD", "0") not in ("1", "true", "True"):
+        return {"ok": True, "model": "local-ollama", "error": None}
+
+    groq_key = os.environ.get("GROQ_API_KEY")
+    if not groq_key:
+        return {"ok": False, "model": "none", "error": "GROQ_API_KEY is not set"}
+
+    groq_model = os.environ.get("GROQ_VISION_MODEL", "qwen/qwen3.8-27b")
+    try:
+        from groq import Groq
+
+        client = Groq(api_key=groq_key)
+        # Try a minimal text-only call to verify model exists and is accessible
+        response = client.chat.completions.create(
+            model=groq_model,
+            messages=[{"role": "user", "content": "ping"}],
+            max_tokens=1,
+            temperature=0,
+        )
+        return {"ok": True, "model": groq_model, "error": None}
+    except Exception as e:
+        return {"ok": False, "model": groq_model, "error": str(e)}

Agent/worker.py

@@ -0,0 +1,494 @@
+import os
+import asyncio
+import io
+import sys
+import contextlib
+from fastapi import FastAPI, HTTPException
+from pydantic import BaseModel
+from typing import Optional
+from fastapi import File, UploadFile
+import base64
+import uuid
+import uvicorn
+
+# Worker that imports the agent and exposes a simple HTTP API
+app = FastAPI(title="Agent Worker")
+
+# ---------------------------------------------------------------------------
+# Schema profile registry (dynamic multi-DB support)
+# ---------------------------------------------------------------------------
+try:
+    from .schema_profile import ProfileRegistry
+except Exception:
+    import Agent.schema_profile as _sp
+    ProfileRegistry = _sp.ProfileRegistry
+
+SCHEMAS_DIR = os.path.join("uploads", "schemas")
+PROFILE_STORE_PATH = os.path.join("chat_store", "schema_profiles.json")
+profile_registry = ProfileRegistry(PROFILE_STORE_PATH)
+
+SUPPORTED_DB_SUFFIXES = {".sqlite", ".db", ".sqlite3", ".csv"}
+
+
+class AddSchemaJsonReq(BaseModel):
+    name: str
+    connection_url: str
+    activate: Optional[bool] = True
+
+
+def _detect_demo_kind(name: str, tblset: Optional[set] = None) -> str:
+    """Classify a profile as the built-in employee demo or a user schema."""
+    n = (name or "").lower()
+    if "demo_employee" in n:
+        return "demo"
+    if tblset and all(
+        t in tblset for t in ["employee", "department", "salary", "dept_emp"]
+    ):
+        return "demo"
+    # Connection URL pointing at employee-ish database
+    if "employee" in (name or ""):
+        return "demo"
+    return "user"
+
+class QueryReq(BaseModel):
+    question: str
+    doc_id: Optional[str] = None
+    mode: Optional[str] = None  # 'text2sql' or 'ocr_qa'
+
+
+class UploadResp(BaseModel):
+    doc_id: str
+    text_preview: str
+    image_url: str
+
+class OcrQAReq(BaseModel):
+    question: str
+    doc_id: str
+
+class OcrQAResp(BaseModel):
+    doc_id: str
+    answer: str
+    used_llm: bool
+
+# Lazy import agent module (do at startup so heavy init happens in worker)
+agent = None
+# Lock to prevent concurrent capture of stdout/stderr which would interleave logs
+capture_lock = asyncio.Lock()
+
+@app.on_event("startup")
+async def startup_event():
+    global agent
+    # Import here so the LLM and DB init occur inside the worker process
+    # Use package-relative import so we load Agent.agentic_workflow (not a top-level file)
+    try:
+        from . import agentic_workflow as agentic_workflow
+    except Exception:
+        # fallback to absolute package import
+        import Agent.agentic_workflow as agentic_workflow
+    agent = agentic_workflow
+
+    # ------------------------------------------------------------------
+    # Built-in demo employee database: ALWAYS available as a profile.
+    # Sourced from the authentic employees sample dataset bundled in the repo
+    # (employee-sample-database/sqlite/dataset_small/employee.db). Used as the
+    # active database out of the box when no DB_CONNECTION_URL is provided.
+    # ------------------------------------------------------------------
+    demo_db_url = None
+    demo_db_path = os.path.join(
+        os.path.dirname(os.path.dirname(os.path.abspath(__file__))),
+        "employee-sample-database", "sqlite", "dataset_small", "employee.db",
+    )
+    if os.path.isfile(demo_db_path):
+        demo_db_url = f"sqlite:///{demo_db_path.replace(os.sep, '/')}"
+        try:
+            has_demo = any((p or {}).get("kind") == "demo"
+                           for p in profile_registry.profiles.values())
+            if not has_demo:
+                profile_registry.add(name="demo_employee", db_url=demo_db_url,
+                                     kind="demo", activate=False)
+                print("[WORKER] Registered built-in demo profile 'demo_employee'")
+        except Exception as reg_err:
+            print(f"[WORKER] demo profile registration skipped: {reg_err}")
+    else:
+        print(f"[WORKER] Bundled demo employee DB not found at {demo_db_path}")
+
+    # ------------------------------------------------------------------
+    # Decide which database to initialize against:
+    #   1. DB_CONNECTION_URL (env/compose wins)  2. last active profile  3. demo
+    # ------------------------------------------------------------------
+    db_url = os.environ.get("DB_CONNECTION_URL")
+    active = profile_registry.active()
+    target_uri = db_url or (active or {}).get("db_url") or demo_db_url
+
+    if target_uri:
+        try:
+            await agent.initialize_database(uri=target_uri)
+            # Keep the registry's active entry in sync with what we connected to.
+            try:
+                if profile_registry.active_id is None:
+                    prof = getattr(agent, "ACTIVE_SCHEMA_PROFILE", None)
+                    _kind = "demo" if (prof and prof.is_employee_db) else "user"
+                    pid = f"boot_{uuid.uuid4().hex[:8]}"
+                    profile_registry.profiles[pid] = {
+                        "name": (prof.name if prof else "startup_db") or "startup_db",
+                        "db_url": target_uri,
+                        "kind": _kind,
+                        "created_at": __import__("time").time(),
+                    }
+                    profile_registry.active_id = pid
+                    profile_registry.save()
+                    print(f"[WORKER] Registered active schema profile '{pid}' for {target_uri}")
+                else:
+                    # point the active id at the profile that owns target_uri
+                    entries = profile_registry.list()
+                    match = next((p for p in entries if (p.get("db_url") or "").replace("\\", "/") == target_uri.replace("\\", "/")), None)
+                    if match and profile_registry.active_id != match["id"]:
+                        profile_registry.switch(match["id"])
+                        print(f"[WORKER] Active profile switched to '{match['name']}'")
+            except Exception as reg_err:
+                print(f"[WORKER] profile bootstrap skipped: {reg_err}")
+        except Exception as e:
+            # Log but continue; agentic_workflow has fallback behavior
+            print(f"Worker: initialize_database failed: {e}")
+    else:
+        print("Worker: no database available (no env DB, no profiles, no demo bundle)")
+    # Optional Groq Vision health check in cloud mode
+    try:
+        from .tools import groq_vision_health
+        hc = groq_vision_health.health_check()
+        if not hc.get("ok", True):
+            print(f"[WORKER] Groq Vision health warning: {hc.get('error')} (model={hc.get('model')})")
+        else:
+            print(f"[WORKER] Groq Vision health: ok (model={hc.get('model')})")
+    except Exception as e:
+        print(f"[WORKER] Groq Vision health check skipped: {e}")
+
+@app.get("/health")
+async def health():
+    return {"status": "ok"}
+
+
+@app.post("/query")
+async def query(req: QueryReq):
+    global agent
+    if agent is None:
+        raise HTTPException(status_code=503, detail="Agent not initialized")
+    q = req.question
+    doc_id = getattr(req, 'doc_id', None)
+    mode = getattr(req, 'mode', None)
+    # Capture stdout/stderr produced by the agent for this request
+    buf_out = io.StringIO()
+    buf_err = io.StringIO()
+    async with capture_lock:
+        try:
+            with contextlib.redirect_stdout(buf_out), contextlib.redirect_stderr(buf_err):
+                # Robust mode switch: if mode is 'ocr_qa' or doc_id is present, use OCR agent; else use SQL logic
+                if (mode == 'ocr_qa') or (doc_id and (mode is None or mode == 'auto')):
+                    # Use OCR Q&A agent
+                    # ocr_agent_qa may be sync, so run in executor
+                    import asyncio
+                    answer = await asyncio.get_event_loop().run_in_executor(
+                        None, lambda: agent.ocr_agent_qa(q, doc_id)
+                    )
+                    result = answer
+                    sql_out = None  # No SQL for OCR Q&A
+                else:
+                    # Use standard Text-to-SQL agentic workflow
+                    result = await agent.agentic_query_process(q, None)
+                    try:
+                        sql_out = getattr(agent, 'LAST_GENERATED_SQL', None)
+                    except Exception:
+                        sql_out = None
+        except ValueError as ve:
+            # Treat known generation failures as normal answers so UI doesn't show nested JSON error
+            err_msg = str(ve)
+            if "Unable to generate SQL query" in err_msg:
+                result = err_msg  # Provide user-visible plain text
+                sql_out = None
+            else:
+                import traceback
+                traceback.print_exc(file=buf_err)
+                raise HTTPException(status_code=500, detail=err_msg)
+        except Exception as e:
+            import traceback
+            traceback.print_exc(file=buf_err)
+            raise HTTPException(status_code=500, detail=str(e))
+    logs = buf_out.getvalue()
+    errlogs = buf_err.getvalue()
+    if errlogs:
+        logs = logs + "\n" + errlogs
+    # Chart contract: strip the embedded chart marker from the answer and
+    # expose it as a first-class field for the web UI (ECharts).
+    chart_out = None
+    try:
+        from .chart_contract import extract_chart_marker
+        clean_answer, chart_out = extract_chart_marker(result)
+        if chart_out is not None:
+            result = clean_answer
+    except Exception:
+        chart_out = None
+    return {"answer": result, "logs": logs, "sql": sql_out, "chart": chart_out}
+
+
+@app.post("/ocr-qa", response_model=OcrQAResp)
+async def ocr_qa(req: OcrQAReq):
+    """Answer a question about a previously uploaded document using its doc_id.
+
+    This uses the agent's ocr_agent_qa helper which pulls stored OCR text and
+    consults the LLM (or returns a low-memory fallback excerpt).
+    """
+    global agent
+    if agent is None:
+        raise HTTPException(status_code=503, detail="Agent not initialized")
+    question = req.question
+    doc_id = req.doc_id
+    # Run potentially blocking OCR QA in thread pool if it's sync
+    try:
+        answer = await asyncio.get_event_loop().run_in_executor(
+            None, lambda: agent.ocr_agent_qa(question, doc_id)
+        )
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=f"ocr_qa failed: {e}")
+    used_llm = not answer.startswith("[Low memory]") and not answer.startswith("No OCR text") and not answer.startswith("Error while")
+    return OcrQAResp(doc_id=doc_id, answer=answer, used_llm=used_llm)
+
+
+@app.post("/upload-image", response_model=UploadResp)
+async def upload_image(file: UploadFile = File(...)):
+    """Accept an image, save locally, call MCP document_scanner.process eagerly,
+    persist OCR to private chat store, and return a doc_id plus preview.
+    """
+    global agent
+    if agent is None:
+        raise HTTPException(status_code=503, detail="Agent not initialized")
+
+    # Ensure uploads directory exists
+    uploads_dir = "uploads"
+    os.makedirs(uploads_dir, exist_ok=True)
+
+    # Save uploaded file
+    filename = f"{uuid.uuid4()}_{file.filename}"
+    path = os.path.join(uploads_dir, filename)
+    try:
+        contents = await file.read()
+        with open(path, "wb") as f:
+            f.write(contents)
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=f"Failed to save upload: {e}")
+
+    # Call MCP document_scanner tool eagerly with base64 payload to avoid exposing file server
+    b64 = base64.b64encode(contents).decode('utf-8')
+    try:
+        from .mcp_client import mcp_call_tool
+        ctx = {"tool_call": {"name": "document_scanner.process", "args": {"image_bytes": b64}}}
+        resp = mcp_call_tool(prompt="run document scanner", timeout=60.0, context=ctx)
+        # Expect tool_result in response
+        tool_result = resp.get("tool_result") if isinstance(resp, dict) else None
+    except Exception:
+        # Fall back to local direct call if MCP unavailable or tool failed
+        tool_result = None
+        try:
+            if agent and hasattr(agent, 'tools') and hasattr(agent.tools, 'document_scanner'):
+                try:
+                    tool_result = agent.tools.document_scanner.process_image(image_bytes=b64)
+                except Exception:
+                    tool_result = None
+        except Exception:
+            tool_result = None
+        if not tool_result:
+            tool_result = {"doc_id": str(uuid.uuid4()), "text": "(ocr unavailable)", "metadata": {}}
+
+    # Persist OCR to private chat store file for retrieval by doc_id
+    doc_id = tool_result.get("doc_id") if isinstance(tool_result, dict) and tool_result.get("doc_id") else str(uuid.uuid4())
+    ocr_text = tool_result.get("text", "") if isinstance(tool_result, dict) else str(tool_result)
+
+    # Save to docs/ and also to chat_store_private for history
+    docs_dir = os.path.join("chat_store", "docs")
+    os.makedirs(docs_dir, exist_ok=True)
+    doc_path = os.path.join(docs_dir, f"{doc_id}.txt")
+    try:
+        with open(doc_path, "w", encoding="utf-8") as df:
+            df.write(ocr_text)
+    except Exception as e:
+        print(f"Failed to persist OCR text: {e}")
+
+    # Also persist the original image path so Granite Vision can do direct visual Q&A
+    img_path_file = os.path.join(docs_dir, f"{doc_id}.img_path")
+    try:
+        abs_img_path = os.path.abspath(path)
+        with open(img_path_file, "w", encoding="utf-8") as ipf:
+            ipf.write(abs_img_path)
+        print(f"[upload] Saved image path for doc_id={doc_id}: {abs_img_path}")
+    except Exception as e:
+        print(f"[upload] Failed to persist image path: {e}")
+
+    # Also add a short entry to chat_store_private for traceability
+    try:
+        try:
+            from .agentic_workflow import chat_store_private, ChatMessage, MessageRole
+        except ImportError:
+            from Agent.agentic_workflow import chat_store_private, ChatMessage, MessageRole
+        # Create minimal messages
+        user_msg = ChatMessage(role=MessageRole.USER, content=f"Uploaded document {doc_id}")
+        assistant_msg = ChatMessage(role=MessageRole.ASSISTANT, content=f"OCR stored: {ocr_text[:400]}")
+        chat_store_private.add_message(key="conversation", message=user_msg)
+        chat_store_private.add_message(key="conversation", message=assistant_msg)
+        chat_store_private.persist(str("chat_store/chat_store_private.json"))
+    except Exception:
+        pass
+
+    # Return doc id and small preview
+    preview = ocr_text[:400]
+    image_url = path
+    return UploadResp(doc_id=doc_id, text_preview=preview, image_url=image_url)
+
+
+# ---------------------------------------------------------------------------
+# Schema profile endpoints — dynamic multi-database support
+# ---------------------------------------------------------------------------
+
+@app.get("/schemas")
+async def schemas_list():
+    return {"profiles": profile_registry.list(), "active_id": profile_registry.active_id}
+
+
+@app.post("/schemas/add")
+async def schemas_add_json(req: AddSchemaJsonReq):
+    """Register a database via connection URL (postgres/mysql/sqlite)."""
+    global agent
+    name = (req.name or "").strip()[:80]
+    if not name or not req.connection_url:
+        raise HTTPException(status_code=400, detail="name and connection_url are required")
+    entry = profile_registry.add(
+        name=name,
+        db_url=req.connection_url,
+        kind=_detect_demo_kind(name),
+        activate=bool(req.activate),
+    )
+    if bool(req.activate) and agent is not None:
+        try:
+            await agent.initialize_database(uri=req.connection_url)
+        except Exception as e:
+            raise HTTPException(status_code=400, detail=f"Profile saved, but DB init failed: {e}")
+    # Return full schema digest preview
+    return {"profile": entry, "schema": agent.ACTIVE_SCHEMA_PROFILE.summarize() if agent else None}
+
+
+@app.post("/schemas/add-file")
+async def schemas_add_file(file: UploadFile = File(...), name: Optional[str] = None, activate: bool = True):
+    """Register a database via file upload (.sqlite/.db/.sqlite3 or .csv)."""
+    global agent
+    fname = file.filename or "upload"
+    ext = os.path.splitext(fname)[-1].lower()
+    if ext not in SUPPORTED_DB_SUFFIXES:
+        raise HTTPException(
+            status_code=400,
+            detail=f"Unsupported file type '{ext}'. Supported: {sorted(SUPPORTED_DB_SUFFIXES)}",
+        )
+    display_name = (name or os.path.splitext(fname)[0]).strip()[:80] or "user_schema"
+
+    os.makedirs(SCHEMAS_DIR, exist_ok=True)
+
+    if ext == ".csv":
+        # Convert CSV into a single-table SQLite database
+        import pandas as pd
+        import tempfile
+
+        sqlite_path = os.path.join(SCHEMAS_DIR, f"{uuid.uuid4().hex}.db")
+        try:
+            contents = await file.read()
+            df = pd.read_csv(io.BytesIO(contents))
+        except Exception as e:
+            raise HTTPException(status_code=400, detail=f"CSV parse failed: {e}")
+        table_name = "".join(ch for ch in display_name if ch.isalnum() or ch == "_") or "data"
+        try:
+            engine_path = os.path.abspath(sqlite_path).replace("\\", "/")
+            df.to_sql(table_name, f"sqlite:///{engine_path}", index=False, if_exists="replace")
+        except Exception as e:
+            raise HTTPException(status_code=500, detail=f"CSV→SQLite conversion failed: {e}")
+        db_url = f"sqlite:///{engine_path}"
+        kind = "user"
+    else:
+        # Copy uploaded SQLite file into schemas dir
+        sqlite_path = os.path.join(SCHEMAS_DIR, f"{uuid.uuid4().hex}{ext}")
+        try:
+            contents = await file.read()
+            with open(sqlite_path, "wb") as fh:
+                fh.write(contents)
+        except Exception as e:
+            raise HTTPException(status_code=500, detail=f"Failed to save upload: {e}")
+        engine_path = sqlite_path.replace("\\", "/")
+        db_url = f"sqlite:///{engine_path}"
+        # Poke the file to validate it is really a SQLite database
+        try:
+            import sqlite3
+
+            conn = sqlite3.connect(sqlite_path)
+            tblset = {r[0] for r in conn.execute(
+                "SELECT name FROM sqlite_master WHERE type IN ('table','view')"
+            ).fetchall()}
+            conn.close()
+        except Exception as e:
+            raise HTTPException(status_code=400, detail=f"Invalid SQLite file: {e}")
+        kind = _detect_demo_kind(display_name, tblset)
+
+    entry = profile_registry.add(name=display_name, db_url=db_url, kind=kind, activate=activate)
+    if activate and agent is not None:
+        try:
+            await agent.initialize_database(uri=db_url)
+        except Exception as e:
+            raise HTTPException(status_code=400, detail=f"Profile saved, but DB init failed: {e}")
+    return {"profile": entry, "schema": agent.ACTIVE_SCHEMA_PROFILE.summarize() if agent else None}
+
+
+@app.post("/schemas/switch/{profile_id}")
+async def schemas_switch(profile_id: str):
+    """Switch the active database profile and re-initialize the agent against it."""
+    global agent
+    if agent is None:
+        raise HTTPException(status_code=503, detail="Agent not initialized")
+    entry = profile_registry.switch(profile_id)
+    if not entry:
+        raise HTTPException(status_code=404, detail="Profile not found")
+    # Dispose previous async engine best-effort
+    try:
+        if getattr(agent, "async_engine", None) is not None:
+            await agent.async_engine.dispose()
+    except Exception:
+        pass
+    try:
+        await agent.initialize_database(uri=entry["db_url"])
+    except Exception as e:
+        raise HTTPException(status_code=400, detail=f"Switch failed: {e}")
+    return {"profile": entry, "schema": agent.ACTIVE_SCHEMA_PROFILE.summarize()}
+
+
+@app.delete("/schemas/{profile_id}")
+async def schemas_delete(profile_id: str):
+    entry = profile_registry.get(profile_id)
+    if not entry:
+        raise HTTPException(status_code=404, detail="Profile not found")
+    # Remove uploaded backing file for file-based profiles
+    db_url = entry.get("db_url") or ""
+    try:
+        if db_url.startswith("sqlite:///"):
+            p = db_url.replace("sqlite:///", "", 1)
+            if os.path.exists(p) and str(SCHEMAS_DIR) in p:
+                os.remove(p)
+    except Exception:
+        pass
+    profile_registry.remove(profile_id)
+    return {"deleted": profile_id}
+
+
+@app.get("/schema/current")
+async def schema_current():
+    """Current active schema summary (tables/columns/samples) for the UI."""
+    global agent
+    if agent is None:
+        raise HTTPException(status_code=503, detail="Agent not initialized")
+    return agent.ACTIVE_SCHEMA_PROFILE.summarize()
+
+if __name__ == '__main__':
+    port = int(os.environ.get("WORKER_PORT", 8700))
+    uvicorn.run(app, host="127.0.0.1", port=port)

Dockerfile

@@ -0,0 +1,42 @@
+# ☁️ Dockerfile for Hugging Face Spaces — Cloud-Only (Lightweight)
+# Image size: ~500MB (no Ollama, no local models)
+# All LLM inference is handled by Groq API and HF Inference API via USE_HF_CLOUD=1
+FROM python:3.11-slim
+
+# Prevent Python from writing pyc files and buffering stdout/stderr
+ENV PYTHONDONTWRITEBYTECODE=1
+ENV PYTHONUNBUFFERED=1
+
+# Cloud mode: all LLM calls go to Groq / HF Inference — no local Ollama needed
+ENV USE_HF_CLOUD=1
+
+# Install system dependencies (Tesseract for OCR fallback only)
+RUN apt-get update && apt-get install -y \
+    curl \
+    zstd \
+    tesseract-ocr \
+    libtesseract-dev \
+    gcc \
+    libpq-dev \
+    && rm -rf /var/lib/apt/lists/*
+
+WORKDIR /app
+
+# Install Python dependencies
+COPY requirements.txt .
+RUN pip install --no-cache-dir --upgrade pip && \
+    pip install --no-cache-dir -r requirements.txt && \
+    pip install --no-cache-dir llama-index-llms-huggingface-api huggingface_hub
+
+# Copy application code
+COPY . .
+
+# Hugging Face Spaces requires port 7860
+EXPOSE 7860
+
+# Fix line endings and permissions on startup script
+RUN sed -i 's/\r$//' start_hf.sh && chmod +x start_hf.sh
+RUN useradd -m -u 1000 appuser && chown -R appuser:appuser /app 2>/dev/null || true
+USER appuser
+
+CMD ["./start_hf.sh"]

README.md

@@ -1,13 +1,633 @@
 ---
-title: QueryLensV2
-emoji: ⚡
+title: QueryLens V2
+emoji: 🔍
 colorFrom: blue
-colorTo: gray
+colorTo: indigo
 sdk: gradio
-sdk_version: 6.30.0
-python_version: '3.12'
+sdk_version: 5.0.0
 app_file: app.py
 pinned: false
 ---
 
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+
+# 🤖 Agentic Text2SQL & Document VQA
+ Multi-Agent System
+
+![Architecture](https://img.shields.io/badge/Architecture-Multi--Agent-blue)
+![Database](https://img.shields.io/badge/Database-Agnostic-success)
+![LLM](https://img.shields.io/badge/LLMs-Qwen%20%7C%20Ollama-orange)
+![Protocol](https://img.shields.io/badge/Protocol-MCP-purple)
+
+An advanced, hardware-optimized AI pipeline built on a multi-agent orchestration architecture. This system seamlessly integrates natural language Text-to-SQL generation with a Vision-Language Model (VLM) for Optical Character Recognition (OCR) and Visual Question Answering (VQA).
+
+By leveraging the Model Context Protocol (MCP), local LLMs via Ollama, and intelligent memory management, this project achieves production-grade data extraction and database querying running entirely on local hardware within tight memory constraints.
+
+---
+
+## Key Features
+
+- **Multi-Agent Orchestration:** A dynamic routing system featuring specialized agents — MemoryAgent, QueryCraftAgent, and ResultPresenterAgent — each with a clearly defined role in the pipeline.
+- **Dynamic Database Profiles:** Connect any database by connection string, or upload a `.sqlite`/`.db`/`.csv` file directly from the UI. Profiles are registered, switchable, and deletable at runtime via the `Database` panel or the `/api/schemas` REST surface — the built-in employee sample DB ships as one profile (`demo_employee`) among many.
+- **Schema-Aware Generic Agents:** The agent pipeline reads a live introspected schema profile (tables, columns, PK/FK, row counts, sample rows) instead of assuming a fixed schema. The fourteen employee-specific `fast_sql` templates activate only when the employee demo is the active profile; every other database gets generic `schema.list_tables / schema.describe_table / schema.sample_data / sql.build_select` MCP tools and profile-aware prompts, validators, and reducers.
+- **Interactive Charts:** Query results that visualize well (category breakdowns, comparisons, time series) are detected automatically and rendered as ECharts visualizations — pie, bar, or line — in a sepia-toned palette. Chart specs ride in the query response as a first-class `chart` field; detection is conservative and never corrupts a textual answer.
+- **Modern Sepia UI:** A Claude-inspired coffee/sepia interface with serif headings and a warm terracotta accent, wrapping the multi-agent trace panels (Memory / QueryCraft / ResultPresenter), live schema browsing, and charting.
+- **Vision-Language Question Answering:** VQA/OCR extraction from invoices and unstructured documents. Served locally via Ollama (`qwen2-vl:2b`) or — in cloud mode — through Groq's `qwen/qwen3.8-27b` vision model (free tier).
+- **Three-Level Text-to-SQL Engine:** Translates natural language questions into syntactically valid SQL through a progressive reasoning architecture, with automatic self-correction at each escalation level.
+- **Semantic Gatekeeping:** A multi-layer validation pipeline filters hallucinated SQL, undefined aliases, missing joins, and incorrect aggregation patterns before any query reaches the database.
+- **Dynamic VRAM Orchestration:** Custom memory management algorithms automatically load and unload models from GPU VRAM to prevent out-of-memory crashes, enabling concurrent AI workloads on consumer hardware.
+- **Unified MCP Integration:** All external data sources — relational databases and vision models alike — are accessed through a single, uniform Model Context Protocol interface.
+- **Database Agnostic Design:** Architected to interface with PostgreSQL, MySQL, SQLite, MS SQL Server, and Oracle via SQLAlchemy.
+- **Conversational Interface:** A full Web UI supporting natural language interaction with persistent session context.
+
+---
+
+## Dynamic Databases & Visual Analytics
+
+### Database profiles
+
+Any database the worker can reach via SQLAlchemy becomes a first-class profile:
+
+| Endpoint | Method | Purpose |
+|---|---|---|
+| `/api/schemas` | GET | List registered profiles + active id |
+| `/api/schemas/add` | POST | Register via `name` + `connection_url` (postgres/mysql/sqlite) |
+| `/api/schemas/add-file` | POST | Upload `.sqlite`/`.db` (used as-is) or `.csv` (converted to a single-table SQLite DB) |
+| `/api/schemas/switch/{id}` | POST | Activate a profile and re-initialize the agent against it |
+| `/api/schemas/{id}` | DELETE | Remove a profile (+ its uploaded backing file) |
+| `/api/schema/current` | GET | Live summary of the active schema (tables, columns, row counts, sample rows) |
+
+Profiles persist in `chat_store/schema_profiles.json` and survive restarts; uploaded files are stored under `uploads/schemas/`. On startup, the worker auto-registers the `DB_CONNECTION_URL` database as the initial active profile.
+
+### Chart contract
+
+When a query returns rows that visualize well, the ResultPresenter embeds a chart spec (pie/bar/line, labels + numeric series) that the worker exposes as `{"chart": {...}}` alongside the text `answer`. Selection heuristics: date-like first column + numeric series → line; ≤8 categories with a single numeric series → pie; otherwise bar. Detection caps at 60 points and never mutates the answer text.
+
+### Working against the demo employee database
+
+The built-in employee sample (MySQL/PostgreSQL containers in `docker-compose.yml`, or the `demo_employee` profile) keeps all battle-tested heuristics: `to_date = '9999-01-01'` current-row filters, the manager/department join knowledge, the 14 `fast_sql` MCP templates, and the invariant enforcement layer.
+
+---
+
+## System Architecture and Agentic Workflow
+
+The project employs a decoupled, asynchronous microservices architecture. The Web UI communicates with a backend worker, which orchestrates all agent requests through the Model Context Protocol.
+
+### The Three-Agent Pipeline
+
+**1. MemoryAgent**
+
+The MemoryAgent acts as the entry point for all user queries. It performs three functions before any SQL generation occurs: checking whether the current query can be satisfied from historical conversation context, rephrasing ambiguous or under-specified queries based on prior interactions, and routing validated new queries to the appropriate downstream agent. This layer prevents redundant computation and improves response latency for repeated or related questions.
+
+**2. QueryCraftAgent — The Three-Level Text-to-SQL Engine**
+
+The QueryCraftAgent is the core reasoning engine of the system. It implements a progressive escalation architecture that balances inference speed with accuracy:
+
+- **Level 1 — Pure LLM Intelligence:** Zero-shot semantic translation of natural language to SQL using Qwen 2.5 Coder. The generated query passes through a semantic validation pipeline that checks alias correctness, required table joins, aggregation integrity, and department or entity filters before being accepted.
+
+- **Level 2 — LLM with SQL Fixer:** When Level 1 fails validation, the agent enters an autonomous self-correction loop. The SQL Fixer applies a deterministic set of targeted transformations — normalizing aliases, injecting missing to_date filters, repairing phantom column references, and enforcing PostgreSQL-compliant GROUP BY clauses — before re-validating the output.
+
+- **Level 3 — LLM with MCP Schema-Grounded SQL Generation:** The deepest reasoning level, reserved for queries that fail both prior levels. For the built-in employee demo, the agent calls one of fourteen registered `fast_sql` MCP tools, each of which queries the live database schema via SQLAlchemy's inspector to retrieve actual column names and table structures, then programmatically constructs a valid SQL template grounded in the real schema. For any user-registered database, the agent instead grounds itself through the generic `schema.list_tables` / `schema.describe_table` / `schema.sample_data` MCP tools, producing a verified live-schema + sample-data hint that applies to any database. In both cases the template is passed as a verified hint to the LLM, which refines it before the Fixer loop runs. Because the hint is built from the actual schema rather than the LLM's training data, this level eliminates the class of failures caused by hallucinated column names or missing joins.
+
+**3. ResultPresenterAgent**
+
+The ResultPresenterAgent executes the validated SQL query against the target database, translates raw result sets into natural conversational responses, handles execution errors and edge cases, and persists the interaction to the conversation history for future MemoryAgent lookups.
+
+### Multi-Agent Orchestration Flow
+
+```mermaid
+graph TD
+    User([User]) --> Web["Web Application\nPort 8000"]
+
+    Web -->|Image Upload| Worker["Backend Worker\nPort 8700"]
+    Web -->|Text Query| Worker
+
+    subgraph MCP["MCP Tool Server — Port 8701"]
+        DocScan["document_scanner.process\nTesseract OCR"]
+        FastSQL["fast_sql tools\nSchema-Grounded SQL Generation"]
+        VisionQA["granite_vision.qa\nQwen 3.5 2B Vision"]
+    end
+
+    Worker -->|"Upload: MCP tool call"| DocScan
+    DocScan -->|"doc_id + OCR text + image path"| Store[("Document Store\nchat_store/docs/")]
+
+    Worker -->|"OCR Q&A mode\ndoc_id present"| VisionQA
+    Store -->|image path| VisionQA
+    VisionQA --> OllamaV(["Ollama\nQwen 3.5 2B Vision"])
+    OllamaV --> Presenter
+
+    Worker -->|Text Query| Memory["MemoryAgent"]
+    Memory -->|Cached answer| Presenter["ResultPresenterAgent"]
+    Memory -->|New query| QCAgent["QueryCraftAgent"]
+
+    QCAgent -->|"Level 1 and 2\nDirect Ollama call"| OllamaSQL(["Ollama\nQwen 2.5 Coder"])
+    QCAgent -->|"Level 3\nMCP tool call"| FastSQL
+    FastSQL -->|"Schema-validated SQL hint"| OllamaSQL
+    OllamaSQL -->|Raw SQL| Fixer["SQL Fixer +\nSemantic Validator"]
+    Fixer -->|"Validated SQL\nvia SQLAlchemy"| DB[("PostgreSQL / MySQL")]
+
+    DB --> Presenter
+    Presenter -->|Persist to history| Memory
+    Presenter --> Answer(["Natural Language Answer"])
+```
+
+---
+
+## Infrastructure and Deployment Architecture
+
+### Containerized Microservices
+
+The system is implemented as a Dockerized microservices architecture, with each component running as an isolated, independently deployable service. Docker Compose orchestrates three services: the main application and agent worker, the MCP tool server, and the relational databases (PostgreSQL and MySQL). Database containers are provisioned with health checks using `pg_isready` and `mysqladmin ping`, and the application container is configured with `condition: service_healthy` dependencies to eliminate the startup race condition inherent in naive `depends_on` configurations.
+
+### Native-Bridge Hybrid Offloading
+
+For peak inference performance, the system uses a hybrid offloading strategy. The application logic and MCP servers run inside Docker containers for portability and isolation, while Ollama and all GPU inference are offloaded to the native Windows host and accessed by the containers over the WSL2 host bridge network (`OLLAMA_BASE_URL=http://<wsl2-host-ip>:11434`).
+
+This architecture eliminates the virtualization overhead introduced by Docker Desktop's Linux VM layer. By giving the inference engine direct access to NVIDIA CUDA cores and the full system memory pool — rather than a virtualized slice — the 1.5B parameter model achieves response times measured in seconds rather than minutes. The separation also means that GPU memory pressure from Ollama does not compete with the container's allocated memory ceiling.
+
+### Database Health and Startup Reliability
+
+A production-grade startup sequence is enforced through Docker health checks and application-level retry logic. The agent application implements a ten-attempt retry loop with five-second intervals on initialization, including a `SELECT 1` connection verification before the schema is read. This ensures consistent behavior across restarts and eliminates the class of intermittent failures caused by the application reading an empty or partially initialized schema during database startup.
+
+### MCP Tool Server Architecture
+
+The Model Context Protocol server runs as a dedicated microservice on port 8701. Three categories of tools are registered at startup:
+
+**document_scanner.process** handles Tesseract OCR and is called at image upload time, not at query time. It returns a doc_id, the extracted text, and triggers persistence of both the OCR output and the image path to the document store.
+
+**granite_vision.qa** is the direct visual question-answering tool backed by Qwen 3.5 2B Vision via Ollama. It is called when a user submits a question with a doc_id and the original image file is available on disk. The internal tool identifier `granite_vision.qa` was retained from the original design specification; the underlying model was later replaced with Qwen 3.5 2B Vision for improved OCR accuracy and reduced VRAM usage, with no changes required to the agent orchestration code.
+
+**fast_sql tools** are fourteen schema-grounded SQL generation tools used exclusively at Level 3 of the QueryCraftAgent. Each tool queries the live database schema via SQLAlchemy's inspector to retrieve actual column names, then uses a deterministic `_build_sql()` function to construct a valid query, optionally verified by a short LLM call. The result is returned as a schema-validated SQL hint to the QueryCraftAgent, which passes it to the LLM for final refinement. This design means Level 3 failures are caused by query complexity or intent ambiguity, never by hallucinated schema details.
+
+---
+
+## Unified Multimodal Integration via MCP
+
+A defining architectural feature of this system is that OCR and multimodal reasoning are integrated through the same Model Context Protocol used for database access. The Qwen 3.5 2B Vision model is registered as a named MCP tool: `granite_vision.qa`. This internal tool identifier was retained from the original design specification; the underlying engine was subsequently replaced with Qwen 3.5 2B Vision, which demonstrated superior OCR accuracy and a smaller VRAM footprint. Because the tool interface is decoupled from its implementation, this substitution required no changes to the agent orchestration code — only the MCP tool registration was updated.
+
+This design delivers three concrete engineering properties:
+
+**Uniform Protocol:** The orchestrator issues identical-format tool calls regardless of whether the data source is a SQL table or a visual document. There are no special-case code paths for different modalities.
+
+**Modular Replaceability:** The vision model is encapsulated behind its MCP interface. Upgrading to a different vision model — such as a GPT-4V endpoint — requires updating only the tool registration, not any agent logic.
+
+**Cross-Modal Reasoning Capability:** Because both data sources share the same protocol, the agent can compose multi-step reasoning across modalities. For example, the system can extract a value from a document via the vision tool and immediately verify it against a database record via the SQL tool, within a single agentic workflow.
+
+```text
+Multimodal Request Flow
+
+Worker receives query with image path
+        |
+        +-- Image detected --> ocr_agent_qa()
+                                    |
+                                    +-- Unload Qwen 2.5 Coder (free VRAM)
+                                    |
+                                    +-- MCP tool call: granite_vision.qa
+                                                |
+                                            MCP Server (port 8701)
+                                                |
+                                            Qwen 3.5 2B Vision (Ollama)
+                                                |
+                                            Structured answer returned
+        |
+        +-- No image --> Standard Text-to-SQL pipeline (unchanged)
+```
+
+---
+
+## SQL Validation Pipeline
+
+The semantic validation layer is one of the most technically substantive components of the system. Rather than trusting LLM output directly, every generated SQL query passes through a deterministic validation pipeline before execution. Key checks include:
+
+- **Alias Integrity:** All table aliases used in SELECT, WHERE, JOIN, and GROUP BY clauses must be defined in FROM or JOIN declarations. The validator maintains a set of defined and used aliases and rejects queries with undefined references.
+- **Salary Table Recognition:** The `salary` table name was explicitly excluded from the SQL keyword blocklist, as it is both a legitimate table name and a column name in the employee schema. Incorrect inclusion in the keyword set caused the validator to silently discard valid salary table joins.
+- **Department Filter Enforcement:** Queries that reference a specific department by name are validated to confirm that a corresponding WHERE filter exists in the generated SQL. This prevents queries that return all departments when the user asked for one.
+- **PostgreSQL GROUP BY Compliance:** Queries containing aggregate functions are checked to ensure all non-aggregated SELECT columns appear in the GROUP BY clause, enforcing strict PostgreSQL semantics.
+- **Manager Schema Awareness:** Queries involving manager data are validated to confirm use of the `dept_manager` table rather than `dept_emp`. The validation produces a warning rather than a hard rejection, allowing the downstream semantic adjuster to regenerate the correct SQL before the query reaches the database.
+
+---
+
+## Research Methodology: Fine-Tuning and Model Selection
+
+As part of the project's scientific evaluation, a custom model fine-tuning pipeline was developed and benchmarked against the selected production model.
+
+**LoRA Fine-Tuning:** The base Qwen 2.5 3B model was fine-tuned using Low-Rank Adaptation on a specialized Text-to-SQL dataset.
+
+**Weight Merging:** The LoRA adapters were merged back into the base model weights via `merge_qwen_to_ollama.py` and quantized for local deployment via Ollama.
+
+**Comparative Evaluation:** The fine-tuned general model was benchmarked against the zero-shot `qwen2.5-coder:1.5b`, which is pre-trained on a large corpus of code and SQL.
+
+**Engineering Decision:** The benchmark results showed that the Coder model's zero-shot SQL accuracy exceeded that of the general LoRA fine-tune, with faster inference and lower error rates on complex multi-table joins. The production architecture was updated accordingly. This outcome demonstrates a data-driven engineering decision: fine-tuning is not always the optimal path when a domain-specialized pretrained model exists for the target task.
+
+See `Fine_Tuning_Qwen_2_5_3B.ipynb` and the `merged_qwen2_5_3b_text2sql` directory for the complete fine-tuning methodology and benchmark results.
+
+---
+
+## Technology Stack
+
+| Component | Technology |
+|---|---|
+| Core Framework | Python 3.11+, FastAPI, Uvicorn |
+| AI Orchestration | LlamaIndex, Model Context Protocol (MCP) |
+| Local Inference | Ollama (C++ optimized, memory-mapped) |
+| Text-to-SQL Model | qwen2.5-coder:1.5b |
+| Vision/OCR Model | qwen3.5:2b (registered as granite_vision.qa) |
+| Database ORM | SQLAlchemy |
+| Containerization | Docker, Docker Compose |
+| Databases | PostgreSQL, MySQL |
+
+---
+
+## Installation and Setup
+
+### Prerequisites
+
+- Ollama installed and running on the host machine
+- Python 3.11 or higher
+- Docker and Docker Compose
+- A supported database (PostgreSQL or MySQL)
+
+### 1. Pull Required Models
+
+```bash
+ollama pull qwen2.5-coder:1.5b
+ollama pull qwen3.5:2b
+```
+
+### 2. Configure Environment
+
+Install Python dependencies:
+
+```bash
+pip install -r requirements.txt
+```
+
+Create a `.env` file in the `Agent/` directory (see `.env.example` for the full reference).
+
+**Recommended (cloud-only, no local models needed):** use the same HF Inference API configuration as the Hugging Face Spaces deployment:
+
+```env
+DB_CONNECTION_URL=postgresql://user:password@localhost:5432/dbname
+USE_HF_CLOUD=1
+HF_API_TOKEN=hf_xxx
+HF_LLM_MODEL=Qwen/Qwen2.5-Coder-7B-Instruct
+
+# Cloud VQA via Groq (free tier)
+GROQ_API_KEY=gsk_xxx
+GROQ_VISION_MODEL=qwen/qwen3.8-27b
+```
+
+**Local Ollama (optional):** with `USE_HF_CLOUD=0`, the configured Ollama model is used — and if it is not installed while `HF_API_TOKEN` is present, the worker **auto-falls back** to HF Inference (`HF_LLM_MODEL`) instead of failing with 404 handshakes.
+
+### 3. Run Without Docker
+
+Start each microservice in a separate terminal:
+
+```bash
+# Terminal 1 — Backend Worker
+python -u -m uvicorn "Agent.worker:app" --port 8700
+
+# Terminal 2 — MCP Tool Server
+python -m uvicorn "Agent.mcp_server:app" --port 8701
+
+# Terminal 3 — Web UI
+python -m uvicorn "web.app:app" --port 8000
+```
+
+Navigate to `http://localhost:8000` to interact with the system. Use the **Database** panel to connect your own database (connection URL or `.sqlite`/`.csv` upload) — the built-in employee sample is provided as a container-backed profile.
+
+### 4. Run With Docker
+
+Update `OLLAMA_BASE_URL` in `docker-compose.yml` to point to your host machine's Ollama instance, then run:
+
+```bash
+docker-compose up --build demo-agent-app
+```
+
+The database containers include health checks and will be fully initialized before the application starts.
+
+---
+
+## Deployment on HuggingFace Spaces
+
+### Quick Deploy (current, cloud-only)
+
+The app is fully cloud-based (`USE_HF_CLOUD=1`): Text-to-SQL runs on HF Inference
+(`Qwen/Qwen2.5-Coder-7B-Instruct`), VQA on Groq (`qwen/qwen3.8-27b`), and the
+built-in `demo_employee` SQLite profile ships with the repo — no external DB
+service is needed on Spaces.
+
+> **Note:** HF Spaces moved the **Docker SDK to a paid tier** (free CPU 2026).
+> Deploy there with the free **Gradio SDK** instead — `app.py` at the repo root
+> spawns the same three services the Docker image ran (worker `:8700`,
+> MCP `:8701`, sepia web UI `:7860`) and mounts a minimal Gradio anchor page at
+> `/gradio-lp`. The UI at `/` is unchanged.
+
+1. **Create the Space** — huggingface.co → *New Space* → SDK: **Gradio** →
+   blank template → name it (e.g. `querylens`). Hardware: free CPU basic.
+2. **Push the project files** into the Space repo. Easiest on Windows is the
+   bundled helper:
+   ```powershell
+   .\deploy_space.ps1 -SpaceName querylens
+   ```
+   It clones `huggingface.co/spaces/<you>/<name>`, copies the project across
+   (respecting the exclusions: no `.env`, no `chat_store/`, no 350 MB
+   `dataset_full` dumps, no `__pycache__`), merges the HF frontmatter into the
+   Space `README.md` (keeping the generated `sdk_version`), commits and pushes.
+3. **Add Secrets** (Space settings → Variables and Secrets — never commit them):
+   | Secret | Purpose |
+   |---|---|
+   | `HF_API_TOKEN` | HF Inference API — Text-to-SQL (`Qwen/Qwen2.5-Coder-7B-Instruct`) |
+   | `GROQ_API_KEY` | Groq vision VQA (`qwen/qwen3.8-27b`, free tier) |
+   No `DB_CONNECTION_URL` is required — the worker auto-activates the bundled demo.
+   Optional: set `DB_CONNECTION_URL` to a hosted Postgres (e.g. Neon) for a real
+   external database.
+4. **Open the Space** — the sepia UI loads with the demo profile active.
+   The **Database** panel lets you register additional databases (hosted
+   Postgres/MySQL URLs, or uploaded `.sqlite`/`.csv` files) at runtime.
+
+**Local Docker runs still work** for your own machine: `docker build` +
+`docker-compose.yml` are unchanged and `start_hf.sh` serves the identical
+stack — only the Spaces *hosting* path is now the Gradio launcher.
+
+Note: free Spaces sleep after inactivity (cold start ~1–2 min). Uploaded schema
+files and chat history are ephemeral unless you enable *Persistent Storage* in
+the Space settings.
+
+### Historical deployment journey (reference)
+
+This section documents the **complete deployment journey** — from the first proof-of-concept tunnel approach to the final production-ready hybrid architecture. Each phase is explained with commands and rationale so this can serve as a reference for deploying any multi-modal AI application on HuggingFace Spaces.
+
+---
+
+### Understanding HuggingFace Spaces Constraints
+
+Before choosing a deployment strategy, it is essential to understand what the free tier of HuggingFace Spaces provides and restricts:
+
+| Resource | Free Tier |
+|----------|-----------|
+| CPU | 2 vCPUs |
+| RAM | 16 GB |
+| GPU | None |
+| Storage | Ephemeral (wiped on restart) |
+| Public port | 7860 only |
+| Timeout | Space sleeps after inactivity |
+
+The key challenge: **running a local LLM (e.g. `qwen2.5-coder:1.5b`) via Ollama consumes ~1.5–3 GB RAM, leaving limited headroom for vision models and the application stack.** Running a vision-language model such as `qwen3.5:2b` (~3 GB) alongside the text model causes out-of-memory (OOM) failures on the free CPU tier.
+
+---
+
+### Phase 1 — Cloudflare Tunnel: Bridging Local Resources to the Cloud
+
+#### Theory
+
+The simplest approach to deploy a resource-intensive app on HF Spaces is to **not run the heavy models in the cloud at all**. Instead, you run the LLM and VLM locally on your development machine and expose the Ollama API endpoint to the internet via a secure tunnel. The HF Space container merely hosts the web frontend and proxies inference requests back to your local machine.
+
+This approach has zero cloud compute cost and allows full GPU acceleration locally. However, it requires your development machine to remain **online and running at all times**, making it unsuitable for a persistent public demo.
+
+#### Setup
+
+**Step 1: Install Cloudflared on your local machine**
+```bash
+# Windows (via winget)
+winget install Cloudflare.cloudflared
+
+# Or download directly
+# https://github.com/cloudflare/cloudflared/releases
+```
+
+**Step 2: Start Ollama locally**
+```bash
+ollama serve
+# Ollama now listens on http://localhost:11434
+```
+
+**Step 3: Create a public tunnel to your local Ollama port**
+```bash
+cloudflared tunnel --url http://localhost:11434
+```
+
+Cloudflared will print a public HTTPS URL like:
+```
+https://your-random-id.trycloudflare.com
+```
+
+**Step 4: Set the tunnel URL as a secret in your HF Space**
+
+In the HF Space settings under **Variables and Secrets**, add:
+```
+OLLAMA_BASE_URL = https://your-random-id.trycloudflare.com
+```
+
+The application code reads `OLLAMA_BASE_URL` and routes all inference calls through the tunnel to your local GPU.
+
+#### Limitation
+
+- The tunnel URL changes every time `cloudflared` restarts, requiring a secret update.
+- Requires a continuously running local machine.
+- Not viable for a shared capstone demo or public deployment.
+
+---
+
+### Phase 2 — Self-Contained Docker: Running Ollama Inside the Container
+
+#### Theory
+
+The next evolution is to **embed Ollama directly into the Docker image** and pull models at container startup. This makes the deployment fully self-contained — no external machine is needed. The HF Space container becomes a complete inference server.
+
+The challenge is that HF Spaces rebuilds the Docker image on every push but does **not cache downloaded model weights** between restarts (storage is ephemeral). This means every **Factory Reboot** triggers a fresh ~1–3 GB model download, causing a cold-start delay of 2–5 minutes.
+
+To prevent HF's health monitor from timing out the Space during this download, model pulling is performed **asynchronously in the background**, allowing the web UI to become responsive immediately while the model fetches in parallel.
+
+#### Dockerfile Changes
+
+Add Ollama installation to the `Dockerfile`:
+
+```dockerfile
+# Install Ollama binary (pinned to a stable release)
+RUN curl -L https://github.com/ollama/ollama/releases/download/v0.5.11/ollama-linux-amd64.tgz \
+    -o ollama.tgz && \
+    tar -C /usr -xzf ollama.tgz && \
+    rm ollama.tgz
+```
+
+#### Startup Script (`start_hf.sh`)
+
+```bash
+#!/bin/bash
+export HOME=/root
+
+echo "[HF-INIT] Starting Ollama server..."
+ollama serve &
+OLLAMA_PID=$!
+
+# Wait for Ollama to become healthy (up to 20 attempts)
+for i in $(seq 1 20); do
+    if curl -sf http://localhost:11434/api/tags > /dev/null 2>&1; then
+        echo "[HF-INIT] Ollama is ALIVE!"
+        break
+    fi
+    echo "[HF-INIT] Waiting... ($i/20)"
+    sleep 3
+done
+
+# Pull the text model asynchronously (non-blocking)
+echo "[HF-INIT] Pulling qwen2.5-coder:1.5b in background..."
+ollama pull qwen2.5-coder:1.5b &
+
+# Start application services immediately (do not wait for model pull)
+echo "[HF-INIT] Starting Backend Worker..."
+python -m uvicorn Agent.worker:app --host 0.0.0.0 --port 8700 &
+
+echo "[HF-INIT] Starting MCP Tool Server..."
+python -m uvicorn Agent.mcp_server:app --host 0.0.0.0 --port 8701 &
+
+echo "[HF-INIT] Starting Web UI on port 7860..."
+exec python -m uvicorn web.app:app --host 0.0.0.0 --port 7860
+```
+
+The `&` (ampersand) operator sends the model pull to the background. The `exec` on the last line replaces the shell process with the web server, ensuring HF's process monitor tracks the correct PID.
+
+#### Limitation
+
+- **Vision model OOM**: Adding `qwen3.5:2b` alongside `qwen2.5-coder:1.5b` exceeds the 16 GB RAM limit when both are loaded simultaneously.
+- Model weights are re-downloaded on every Factory Reboot.
+- Ollama version pinning is critical — newer models require newer Ollama binaries and pulling the wrong combination causes `412` errors.
+
+---
+
+### Phase 3 — Hybrid Serverless API (Final Production Architecture)
+
+#### Theory
+
+The final and most robust architecture eliminates the in-container LLM entirely for text generation and uses **external serverless inference APIs** instead. The container only runs lightweight services (web UI, API proxy, Tesseract OCR), while all neural inference is delegated to dedicated providers:
+
+| Task | Provider | Model | Cost |
+|------|----------|-------|------|
+| Text-to-SQL | HuggingFace Inference API | `Qwen/Qwen2.5-Coder-7B-Instruct` | Free tier |
+| Vision Q&A | Groq API | `meta-llama/llama-4-scout-17b-16e-instruct` | Free (14,400 req/day) |
+| OCR | Tesseract (in container) | — | Free |
+| Database | Neon PostgreSQL | — | Free tier |
+
+This eliminates OOM risk entirely, removes the model download cold-start, and makes the container startup under 30 seconds.
+
+#### Why Groq for Vision Instead of HuggingFace Serverless?
+
+HuggingFace's serverless inference for vision-language models has two limitations on the free tier:
+1. Most VLM models are not routed through any enabled provider for free-tier accounts.
+2. Google Gemini's free tier blocks requests originating from cloud datacenter IP ranges (such as HF's infrastructure), setting the effective quota to zero even for valid free-tier API keys.
+
+**Groq** is purpose-built for API-first server deployments and explicitly allows free-tier requests from cloud IPs. It provides 14,400 free requests per day with no IP restrictions.
+
+#### Required HuggingFace Space Secrets
+
+Set these in **Space Settings → Variables and Secrets**:
+
+```
+USE_HF_CLOUD       = 1
+HF_API_TOKEN       = # HuggingFace access token
+GROQ_API_KEY       = # From console.groq.com (free)
+DB_CONNECTION_URL  = # Neon PostgreSQL connection string
+```
+
+#### Obtaining API Keys
+
+**HuggingFace Token:**
+1. Go to https://huggingface.co/settings/tokens
+2. Create a token with `read` scope (or `write` if you need to push models)
+
+**Groq API Key:**
+1. Go to https://console.groq.com
+2. Sign in with Google / GitHub
+3. Navigate to **API Keys** → **Create API Key**
+4. Copy the key
+
+**Neon PostgreSQL:**
+1. Go to https://neon.tech
+2. Create a free project
+3. Copy the connection string from the dashboard
+
+#### Slim Dockerfile (No Ollama Required)
+
+```dockerfile
+FROM python:3.11-slim
+
+ENV PYTHONDONTWRITEBYTECODE=1
+ENV PYTHONUNBUFFERED=1
+ENV USE_HF_CLOUD=1
+
+RUN apt-get update && apt-get install -y \
+    curl \
+    tesseract-ocr \
+    libtesseract-dev \
+    gcc \
+    libpq-dev \
+    && rm -rf /var/lib/apt/lists/*
+
+WORKDIR /app
+COPY requirements.txt .
+RUN pip install --no-cache-dir -r requirements.txt
+
+COPY . .
+RUN useradd -m -u 1000 appuser && chown -R appuser:appuser /app
+USER appuser
+
+CMD ["bash", "start_hf.sh"]
+```
+
+#### Key `requirements.txt` Additions
+
+```txt
+huggingface_hub      # HF Inference API client
+groq>=0.9.0          # Groq Vision API client
+google-genai>=1.0.0  # Optional: Google Gemini (requires billing for cloud IPs)
+pytesseract>=0.3.10  # Local OCR fallback
+pillow>=10.0.0
+```
+
+#### Deployment Steps
+
+1. **Create a HuggingFace Space** at https://huggingface.co/spaces
+   - SDK: **Docker**
+   - Visibility: Public or Private
+
+2. **Push your repository** to the Space's git remote:
+   ```bash
+   git remote add space https://huggingface.co/spaces/<your-username>/<space-name>
+   git push space main
+   ```
+
+3. **Add secrets** in Space Settings (see table above). Do **not** hardcode secrets in the Dockerfile or any committed file.
+
+4. **Factory Reboot** the Space after adding secrets to trigger a clean rebuild with the new environment variables.
+
+5. **Monitor the build logs** — a successful startup shows:
+   ```
+   [GRANITE_VQA] Target Model: meta-llama/llama-4-scout-17b-16e-instruct (via Groq Vision)
+   [INFO] CLOUD MODE: Initializing Official HF InferenceClient (Qwen/Qwen2.5-Coder-7B-Instruct)
+   INFO: Uvicorn running on http://0.0.0.0:7860
+   ```
+
+#### Important Notes on Line Endings
+
+HuggingFace containers run Linux. If you develop on Windows, your shell scripts will have `\r\n` (CRLF) line endings, which cause `bad interpreter` errors on Linux. Add this to your `Dockerfile` to strip them automatically:
+
+```dockerfile
+RUN sed -i 's/\r$//' start_hf.sh && chmod +x start_hf.sh
+```
+
+Alternatively, configure Git to normalize line endings:
+```bash
+git config core.autocrlf input
+```
+
+
+---
+
+## Project Summary
+
+This project demonstrates that a self-correcting, multi-modal AI pipeline can be built and deployed entirely on consumer hardware through careful architectural decisions: progressive SQL generation with semantic validation, unified MCP-based tool access across data modalities, dynamic VRAM orchestration to prevent memory exhaustion, and a hybrid containerization strategy that preserves native GPU performance while maintaining deployment portability.
+
+The system was developed iteratively, with each architectural decision — from model selection to validation logic to infrastructure design — informed by observed failure modes and measured outcomes rather than theoretical assumptions.
+
+---
+*Built with ❤️ for advanced Agentic Data Engineering.*

app.py

@@ -0,0 +1,72 @@
+"""
+Hugging Face Spaces entrypoint — Gradio SDK wrapper
+====================================================
+
+HF Spaces moved the Docker SDK to a paid tier, so this Gradio-SDK Space
+replicates the exact same three-service architecture the Docker image used:
+
+    this process (port 7860, public)
+      ├── Agent.worker      → spawned uvicorn :8700  (Text2SQL / DocVQA backend)
+      ├── Agent.mcp_server  → spawned uvicorn :8701  (MCP tool server)
+      └── web.app FastAPI   → serves the sepia UI + /api proxies on :7860
+
+A minimal Gradio Blocks page is mounted at ``/gradio-lp`` purely so the
+Gradio-SDK runtime sees a registered Blocks application — the real UI is
+the sepia web app at ``/``.
+
+Secrets (Space settings → Variables and Secrets, never baked into the code):
+    HF_API_TOKEN   — HF Inference API, Text-to-SQL on
+                     Qwen/Qwen2.5-Coder-7B-Instruct
+    GROQ_API_KEY   — Groq vision VQA (free tier)
+
+No DB_CONNECTION_URL is required — the bundled authentic demo employee
+SQLite database (employee-sample-database/sqlite/dataset_small/employee.db)
+auto-activates as the demo_employee profile.
+"""
+import os
+import sys
+import subprocess
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parent
+
+
+# -----------------------------------------------------------------------------
+# Background services (same as start_hf.sh in the Docker image)
+# -----------------------------------------------------------------------------
+def _spawn(module: str, port: int, name: str) -> subprocess.Popen:
+    """Start a uvicorn service; output streams into the Space log viewer."""
+    print(f"[LAUNCHER] starting {name} on :{port} ...", flush=True)
+    return subprocess.Popen(
+        [sys.executable, "-u", "-m", "uvicorn", module,
+         "--host", "0.0.0.0", "--port", str(port)],
+        cwd=str(ROOT),
+    )
+
+
+_worker = _spawn("Agent.worker:app", 8700, "backend worker")
+_mcp = _spawn("Agent.mcp_server:app", 8701, "MCP tool server")
+
+# -----------------------------------------------------------------------------
+# Web UI: our existing FastAPI app, with a tiny Gradio shell mounted beside it
+# so the Gradio-SDK runtime is satisfied.
+# -----------------------------------------------------------------------------
+import gradio as gr  # noqa: E402  (preinstalled in the Gradio-SDK runtime)
+from web.app import app as fastapi_app  # noqa: E402
+
+with gr.Blocks() as _lp:
+    gr.Markdown(
+        "**QueryLens** — the full sepia UI is served at this Space's root `/`.\n\n"
+        "This page is only an anchor for the Gradio runtime."
+    )
+    gr.HTML('<a href="/" style="text-decoration:none;font-weight:600;">← Open the QueryLens UI</a>')
+
+app = gr.mount_gradio_app(fastapi_app, _lp, path="/gradio-lp")
+
+
+if __name__ == "__main__":
+    import uvicorn
+
+    port = int(os.environ.get("PORT", "7860"))
+    print(f"[LAUNCHER] QueryLens web UI live on http://0.0.0.0:{port}", flush=True)
+    uvicorn.run(app, host="0.0.0.0", port=port)

docker-compose.yml

@@ -0,0 +1,84 @@
+version: '3.8'
+
+services:
+  # Main Agentic Application (worker :8700, MCP tool server :8701, web UI :7860)
+  demo-agent-app:
+    build: .
+    container_name: demo-agent-app
+    restart: unless-stopped
+    env_file: Agent/.env
+    environment:
+      - OLLAMA_BASE_URL=http://172.27.224.1:11434
+      - LLM_CTX=4096
+    deploy:
+      resources:
+        limits:
+          cpus: '2.0'
+          memory: 4G
+    ports:
+      - "7860:7860"   # Web UI (QueryLens)
+      - "8700:8700"   # Worker API
+      - "8701:8701"   # MCP tool server
+    depends_on:
+      demo-postgres:
+        condition: service_healthy
+      demo-mysql:
+        condition: service_healthy
+    volumes:
+      - ./Agent:/app/Agent
+      - ./web:/app/web
+      # Dynamic database profiles: uploaded schema files + profile registry
+      - demo-schema-uploads:/app/uploads/schemas
+      - demo-chat-store:/app/chat_store
+    networks:
+      - demo-net
+
+  # MySQL Sample Database (built-in employee demo)
+  demo-mysql:
+    image: mysql:8.0
+    container_name: demo-mysql
+    restart: unless-stopped
+    env_file: Agent/.env
+    ports:
+      - "${MYSQL_PORT:-3306}:3306"
+    volumes:
+      - ./employee-sample-database/mysql/dataset_small:/docker-entrypoint-initdb.d
+      - demo-mysql-data:/var/lib/mysql
+    networks:
+      - demo-net
+    healthcheck:
+      test: ["CMD", "mysqladmin", "ping", "-h", "localhost", "--silent"]
+      interval: 5s
+      timeout: 5s
+      retries: 12
+      start_period: 30s
+
+  # PostgreSQL Sample Database (built-in employee demo)
+  demo-postgres:
+    image: postgres:15
+    container_name: demo-postgres
+    restart: unless-stopped
+    env_file: Agent/.env
+    ports:
+      - "${POSTGRES_PORT:-5432}:5432"
+    volumes:
+      - ./employee-sample-database/postgres/dataset_small:/docker-entrypoint-initdb.d
+      - demo-postgres-data:/var/lib/postgresql/data
+    networks:
+      - demo-net
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -U $${POSTGRES_USER:-postgres}"]
+      interval: 5s
+      timeout: 5s
+      retries: 12
+      start_period: 30s
+
+volumes:
+  demo-mysql-data:
+  demo-postgres-data:
+  demo-schema-uploads:
+  demo-chat-store:
+
+networks:
+  demo-net:
+    driver: bridge

requirements.txt

@@ -0,0 +1,40 @@
+# Core Frameworks
+fastapi>=0.100.0
+uvicorn>=0.23.0
+sqlalchemy>=2.0.0
+pydantic>=2.0.0
+python-multipart>=0.0.6
+pandas>=2.0.0
+
+# Multi-Agent Orchestration
+llama-index>=0.10.0
+llama-index-core>=0.10.0
+llama-index-llms-ollama>=0.1.2
+llama-index-llms-huggingface-api
+huggingface_hub
+
+# Model Context Protocol
+mcp>=0.1.0
+sqlparse>=0.4.4
+
+# HTTP & Async
+requests>=2.31.0
+httpx>=0.26.0
+aiohttp>=3.9.0
+psutil>=5.9.0
+
+# Database Drivers
+pymysql>=1.1.0
+asyncpg>=0.29.0
+psycopg2-binary>=2.9.0
+aiomysql>=0.2.0
+aiosqlite>=0.19.0
+
+# Document Vision processing
+pillow>=10.0.0
+pytesseract>=0.3.10
+google-genai>=1.0.0
+groq>=0.9.0
+
+# Environment
+python-dotenv>=1.0.0

start_hf.sh

@@ -0,0 +1,24 @@
+#!/bin/bash
+
+# ☁️ Cloud-Only Startup Script for Hugging Face Spaces
+# All LLM calls are routed to Groq and HF Inference APIs via USE_HF_CLOUD=1
+# Ollama is NOT used — image is kept lightweight to prevent HF storage purges.
+
+echo "[HF-INIT] ===== Application Startup at $(date -u '+%Y-%m-%d %H:%M:%S') ====="
+
+# 1. Start Backend Worker
+echo "[HF-INIT] Starting Backend Worker on port 8700..."
+python -u -m uvicorn "Agent.worker:app" --host 0.0.0.0 --port 8700 &
+
+# 2. Start MCP Tool Server
+echo "[HF-INIT] Starting MCP Tool Server on port 8701..."
+python -m uvicorn "Agent.mcp_server:app" --host 0.0.0.0 --port 8701 &
+
+# 3. Start Web UI
+echo "[HF-INIT] Starting Web UI on port 7860..."
+python -m uvicorn "web.app:app" --host 0.0.0.0 --port 7860 &
+
+echo "[HF-INIT] All services started. Container is live."
+
+# Keep the container alive
+wait

upload_binaries.py

@@ -0,0 +1,73 @@
+"""
+upload_binaries.py -- push binary files to a HF Space via the Xet backend.
+
+HF Spaces now reject binary files on plain `git push`; the supported path is
+`huggingface_hub` (>= 0.32 with `hf_xet`), which uploads through Xet storage.
+
+Usage:
+    python upload_binaries.py -r OWNER/REPO -p path/in/repo.db -f local/file.db
+
+Auth: uses your cached HF token (created by `huggingface-cli login`).
+      Skips the upload if the remote file already has identical bytes.
+"""
+import argparse
+import hashlib
+import sys
+
+from huggingface_hub import HfApi, hf_hub_download, get_token
+
+
+def sha256_of(path: str) -> str:
+    h = hashlib.sha256()
+    with open(path, "rb") as fh:
+        for chunk in iter(lambda: fh.read(1 << 20), b""):
+            h.update(chunk)
+    return h.hexdigest()
+
+
+def main() -> int:
+    ap = argparse.ArgumentParser()
+    ap.add_argument("-r", "--repo", required=True, help="repo id, e.g. OWNER/SPACE")
+    ap.add_argument("-p", "--path-in-repo", required=True)
+    ap.add_argument("-f", "--file", required=True, help="local file to upload")
+    args = ap.parse_args()
+
+    if not get_token():
+        print("ERROR: no HF token cached. Run 'huggingface-cli login' (token needs write access) and retry.")
+        return 2
+
+    api = HfApi()
+    repo_type = "space"  # this script targets Spaces ("OWNER/SPACE" ids)
+
+    remote = None
+    try:
+        remote = hf_hub_download(
+            repo_id=args.repo, repo_type=repo_type, filename=args.path_in_repo
+        )
+    except Exception:
+        remote = None
+
+    local_hash = sha256_of(args.file)
+    if remote is not None:
+        try:
+            if sha256_of(remote) == local_hash:
+                print(f"[XET] {args.path_in_repo} already identical on {args.repo} - skipping upload")
+                return 0
+            print(f"[XET] {args.path_in_repo} differs on the remote {args.repo} - uploading new revision")
+        except OSError:
+            pass
+
+    print(f"[XET] uploading {args.path_in_repo} ({args.file}) to {args.repo} ...")
+    api.upload_file(
+        repo_id=args.repo,
+        repo_type=repo_type,
+        path_in_repo=args.path_in_repo,
+        path_or_fileobj=args.file,
+        commit_message=f"Add binary demo database via Xet: {args.path_in_repo}",
+    )
+    print(f"[XET] uploaded OK (sha256 {local_hash[:12]}...)")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

web/app.py

@@ -0,0 +1,283 @@
+import os
+import httpx
+import logging
+import traceback
+from typing import Optional
+from fastapi import FastAPI, HTTPException, Request
+from pydantic import BaseModel
+from fastapi.staticfiles import StaticFiles
+from fastapi.responses import FileResponse, JSONResponse
+from fastapi.middleware.cors import CORSMiddleware
+
+WORKER_URL = os.environ.get("WORKER_URL", "http://localhost:8700")
+# Timeout (seconds) when calling the worker; re-execute and heavy queries can take longer.
+# Make configurable via environment variable WORKER_TIMEOUT (default 300s).
+WORKER_TIMEOUT = float(os.environ.get("WORKER_TIMEOUT", "450"))  # Increased to 450s to handle complex queries
+
+# Reusable AsyncClient to avoid per-request connection setup overhead
+client: httpx.AsyncClient | None = None
+
+app = FastAPI(title="Text2SQL Web API")
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=["*"],  # adjust in production
+    allow_credentials=True,
+    allow_methods=["*"],
+    allow_headers=["*"],
+)
+
+# Serve static frontend under /static and provide a root route
+static_dir = os.path.join(os.path.dirname(__file__), 'static')
+if os.path.isdir(static_dir):
+    app.mount("/static", StaticFiles(directory=static_dir, html=True), name="static")
+
+
+@app.get("/")
+async def root_index():
+    index_path = os.path.join(static_dir, 'index.html')
+    return FileResponse(index_path)
+
+class QueryReq(BaseModel):
+    question: str
+    # Optional doc_id produced by a prior /upload-image call
+    doc_id: Optional[str] = None
+    # Explicit mode hint from client: 'text2sql' or 'ocr_qa'. When omitted, worker may auto-detect.
+    mode: Optional[str] = None
+
+class OcrQAReq(BaseModel):
+    question: str
+    doc_id: str
+
+class OcrQAResp(BaseModel):
+    doc_id: str
+    answer: str
+    used_llm: bool
+
+@app.get('/health')
+async def health():
+    return {"status": "ok"}
+
+@app.post('/api/query')
+async def api_query(req: QueryReq):
+    # Use a shared AsyncClient created at startup to avoid TCP/TLS/HTTP handshakes per request.
+    global client
+    if client is None:
+        # fallback to create a temporary client (shouldn't happen if startup ran)
+        client = httpx.AsyncClient(timeout=WORKER_TIMEOUT)
+    try:
+        payload = {"question": req.question}
+        # Forward explicit mode if provided so the worker doesn't infer from lingering doc_id
+        if getattr(req, 'mode', None):
+            payload['mode'] = req.mode
+        if getattr(req, 'doc_id', None):
+            payload['doc_id'] = req.doc_id
+        r = await client.post(f"{WORKER_URL}/query", json=payload)
+        r.raise_for_status()
+        return r.json()
+    except httpx.HTTPStatusError as e:
+        # Log the full traceback and response body for debugging
+        logging.error("Worker returned non-2xx status: %s", e.response.status_code)
+        logging.error("Response text: %s", getattr(e.response, 'text', str(e)))
+        logging.error(traceback.format_exc())
+        raise HTTPException(status_code=e.response.status_code, detail=getattr(e.response, 'text', str(e)))
+    except httpx.ReadTimeout as e:
+        # Worker took too long. Return a 504 gateway timeout with a helpful message.
+        logging.error("Timeout contacting worker after %ss: %s", WORKER_TIMEOUT, str(e))
+        logging.error(traceback.format_exc())
+        return JSONResponse(status_code=504, content={
+            "error": "worker_timeout",
+            "message": f"Worker did not respond within {WORKER_TIMEOUT} seconds. The query may be long-running (re-execute) or the worker is busy. Try increasing WORKER_TIMEOUT or running the worker with more resources.",
+        })
+    except Exception as e:
+        logging.error("Error contacting worker: %s", str(e))
+        logging.error(traceback.format_exc())
+        # Return a JSON response containing the error detail to help debugging from the client
+        return JSONResponse(status_code=500, content={"error": str(e), "trace": traceback.format_exc()})
+
+
+@app.post('/api/ocr-qa')
+async def api_ocr_qa(req: OcrQAReq):
+    global client
+    if client is None:
+        client = httpx.AsyncClient(timeout=WORKER_TIMEOUT)
+    try:
+        payload = {"question": req.question, "doc_id": req.doc_id}
+        r = await client.post(f"{WORKER_URL}/ocr-qa", json=payload)
+        r.raise_for_status()
+        return r.json()
+    except httpx.HTTPStatusError as e:
+        logging.error("Worker OCR QA non-2xx: %s", e.response.status_code)
+        logging.error("Response text: %s", getattr(e.response, 'text', str(e)))
+        return JSONResponse(status_code=e.response.status_code, content={"error": getattr(e.response, 'text', str(e))})
+    except Exception as e:
+        logging.error("Error contacting worker OCR QA: %s", str(e))
+        return JSONResponse(status_code=500, content={"error": str(e), "trace": traceback.format_exc()})
+
+
+@app.on_event('startup')
+async def startup_event():
+    global client
+    if client is None:
+        client = httpx.AsyncClient(timeout=WORKER_TIMEOUT)
+
+
+@app.on_event('shutdown')
+async def shutdown_event():
+    global client
+    if client is not None:
+        await client.aclose()
+
+
+@app.post('/api/test')
+async def api_test(req: QueryReq):
+    # Simple echo endpoint to verify POST handling without contacting the worker
+    return {"ok": True, "echo": req.question}
+
+
+@app.post('/upload-image')
+async def upload_image(file: bytes = None, request: Request = None):
+    """Proxy an image upload to the worker's /upload-image endpoint.
+
+    Accepts multipart/form-data file field 'file' from the browser.
+    Returns the worker JSON (doc_id, text_preview, image_url) on success.
+    """
+    # Use the request.form() to access the uploaded file
+    try:
+        form = await request.form()
+        f = form.get('file')
+        if f is None:
+            raise HTTPException(status_code=400, detail='file field is required')
+        # forward the file to the worker
+        files = {'file': (f.filename, await f.read(), f.content_type)}
+        async with httpx.AsyncClient(timeout=120) as c:
+            r = await c.post(f"{WORKER_URL}/upload-image", files=files)
+            r.raise_for_status()
+            return JSONResponse(status_code=200, content=r.json())
+    except HTTPException:
+        raise
+    except Exception as e:
+        logging.error('upload-image proxy failed: %s', str(e))
+        raise HTTPException(status_code=500, detail=str(e))
+
+
+# ---------------------------------------------------------------------------
+# Schema profile proxy endpoints — dynamic multi-database support
+# ---------------------------------------------------------------------------
+
+@app.get('/api/schemas')
+async def api_schemas():
+    global client
+    try:
+        r = await client.get(f"{WORKER_URL}/schemas")
+        r.raise_for_status()
+        return r.json()
+    except httpx.HTTPStatusError as e:
+        return JSONResponse(status_code=e.response.status_code,
+                            content={"error": getattr(e.response, 'text', str(e))})
+    except Exception as e:
+        logging.error('schemas list proxy failed: %s', str(e))
+        return JSONResponse(status_code=500, content={"error": str(e)})
+
+
+class AddSchemaReq(BaseModel):
+    name: str
+    connection_url: str
+    activate: Optional[bool] = True
+
+
+@app.post('/api/schemas/add')
+async def api_schemas_add(req: AddSchemaReq):
+    global client
+    try:
+        r = await client.post(f"{WORKER_URL}/schemas/add", json=req.model_dump(),
+                              timeout=WORKER_TIMEOUT)
+        r.raise_for_status()
+        return r.json()
+    except httpx.HTTPStatusError as e:
+        return JSONResponse(status_code=e.response.status_code,
+                            content={"error": getattr(e.response, 'text', str(e))})
+    except Exception as e:
+        logging.error('schemas add proxy failed: %s', str(e))
+        return JSONResponse(status_code=500, content={"error": str(e)})
+
+
+@app.post('/api/schemas/add-file')
+async def api_schemas_add_file(request: Request):
+    """Proxy a schema file upload (.sqlite/.db/.csv) to the worker."""
+    global client
+    try:
+        form = await request.form()
+        f = form.get('file')
+        if f is None:
+            raise HTTPException(status_code=400, detail='file field is required')
+        data = {'name': form.get('name') or '', 'activate': form.get('activate', 'true')}
+        files = {'file': (f.filename, await f.read(), f.content_type)}
+        # DB init can be slow on first connect; generous timeout
+        async with httpx.AsyncClient(timeout=300) as c:
+            r = await c.post(f"{WORKER_URL}/schemas/add-file", files=files, data=data)
+            r.raise_for_status()
+            return r.json()
+    except HTTPException:
+        raise
+    except httpx.HTTPStatusError as e:
+        return JSONResponse(status_code=e.response.status_code,
+                            content={"error": getattr(e.response, 'text', str(e))})
+    except Exception as e:
+        logging.error('schemas add-file proxy failed: %s', str(e))
+        return JSONResponse(status_code=500, content={"error": str(e)})
+
+
+@app.post('/api/schemas/switch/{profile_id}')
+async def api_schemas_switch(profile_id: str):
+    global client
+    try:
+        r = await client.post(f"{WORKER_URL}/schemas/switch/{profile_id}",
+                              timeout=WORKER_TIMEOUT)
+        r.raise_for_status()
+        return r.json()
+    except httpx.HTTPStatusError as e:
+        return JSONResponse(status_code=e.response.status_code,
+                            content={"error": getattr(e.response, 'text', str(e))})
+    except Exception as e:
+        logging.error('schemas switch proxy failed: %s', str(e))
+        return JSONResponse(status_code=500, content={"error": str(e)})
+
+
+@app.delete('/api/schemas/{profile_id}')
+async def api_schemas_delete(profile_id: str):
+    global client
+    try:
+        r = await client.delete(f"{WORKER_URL}/schemas/{profile_id}")
+        r.raise_for_status()
+        return r.json()
+    except httpx.HTTPStatusError as e:
+        return JSONResponse(status_code=e.response.status_code,
+                            content={"error": getattr(e.response, 'text', str(e))})
+    except Exception as e:
+        logging.error('schemas delete proxy failed: %s', str(e))
+        return JSONResponse(status_code=500, content={"error": str(e)})
+
+
+@app.get('/api/schema/current')
+async def api_schema_current():
+    global client
+    try:
+        r = await client.get(f"{WORKER_URL}/schema/current")
+        r.raise_for_status()
+        return r.json()
+    except httpx.HTTPStatusError as e:
+        return JSONResponse(status_code=e.response.status_code,
+                            content={"error": getattr(e.response, 'text', str(e))})
+    except Exception as e:
+        logging.error('schema current proxy failed: %s', str(e))
+        return JSONResponse(status_code=500, content={"error": str(e)})
+
+
+@app.get('/api/test')
+async def api_test_get():
+    return {"ok": True, "msg": "POST to /api/test with JSON {question: ...} to echo"}
+
+if __name__ == '__main__':
+    import uvicorn
+    port = int(os.environ.get('WEB_PORT', 8000))
+    uvicorn.run(app, host='0.0.0.0', port=port)

web/static/app.js

@@ -0,0 +1,836 @@
+const askBtn = document.getElementById('ask');
+const questionEl = document.getElementById('question');
+const answerEl = document.getElementById('answer');
+const sqlEl = document.getElementById('sql');
+const memoryEl = document.getElementById('memory');
+const querycraftEl = document.getElementById('querycraft');
+const presenterEl = document.getElementById('presenter');
+const uploadInput = document.getElementById('upload');
+const uploadBtn = document.getElementById('uploadBtn');
+const uploadStatus = document.getElementById('uploadStatus');
+const uploadControls = document.getElementById('uploadControls');
+const modeToggle = document.getElementById('modeToggle');
+let currentDocId = null;
+let currentMode = 'text2sql';
+const statusEl = document.getElementById('status');
+const timeEl = document.getElementById('time');
+const recalcBtn = document.getElementById('recalc');
+const spinner = document.getElementById('spinner');
+const spinnerText = document.getElementById('spinnerText');
+const askBtnEl = askBtn;
+
+// Mode toggle logic
+modeToggle.addEventListener('change', function() {
+  currentMode = modeToggle.value;
+  console.log('[DEBUG] Mode changed to:', currentMode); // Debug log
+  if(currentMode === 'ocr_qa') {
+    uploadControls.classList.remove('hidden');
+    console.log('[DEBUG] Upload controls shown'); // Debug log
+  } else {
+    uploadControls.classList.add('hidden');
+    currentDocId = null;
+    uploadStatus.textContent = 'No upload';
+    uploadStatus.className = 'status-text';
+    console.log('[DEBUG] Upload controls hidden'); // Debug log
+  }
+});
+
+// Initialize UI state
+console.log('[DEBUG] Initial mode:', currentMode); // Debug log
+if(currentMode === 'ocr_qa') {
+  uploadControls.classList.remove('hidden');
+} else {
+  uploadControls.classList.add('hidden');
+}
+
+// Helper to programmatically switch to OCR Q&A mode
+function switchToOcrQaMode() {
+  if (modeToggle.value !== 'ocr_qa') {
+    modeToggle.value = 'ocr_qa';
+    currentMode = 'ocr_qa';
+    uploadControls.classList.remove('hidden');
+    // Fire a change event so any listeners update state consistently
+    const ev = new Event('change', { bubbles: true });
+    modeToggle.dispatchEvent(ev);
+    console.log('[DEBUG] Switched to OCR Q&A mode');
+  }
+}
+
+// Extract mode change logic into a function so it can be called directly
+function onModeChange() {
+  currentMode = modeToggle.value;
+  console.log('[DEBUG] onModeChange called, mode:', currentMode); // Debug log
+  if(currentMode === 'ocr_qa') {
+    uploadControls.classList.remove('hidden');
+    console.log('[DEBUG] Upload controls shown via onModeChange'); // Debug log
+  } else {
+    uploadControls.classList.add('hidden');
+    currentDocId = null;
+    uploadStatus.textContent = 'No upload';
+    uploadStatus.className = 'status-text';
+    console.log('[DEBUG] Upload controls hidden via onModeChange'); // Debug log
+  }
+}
+
+// Mode toggle logic
+modeToggle.addEventListener('change', onModeChange);
+
+// Upload handler: create a temporary file input on demand to avoid issues with a permanently hidden input.
+if (uploadBtn) {
+  uploadBtn.addEventListener('click', () => {
+    // If not in OCR Q&A mode, switch and highlight, then open file picker after a delay
+    if (modeToggle.value !== 'ocr_qa') {
+      switchToOcrQaMode();
+      // Visually highlight the dropdown to show the switch
+      modeToggle.style.boxShadow = '0 0 0 3px rgba(15, 98, 254, 0.2)';
+      modeToggle.style.transition = 'box-shadow 0.2s';
+      setTimeout(() => {
+        modeToggle.style.boxShadow = '';
+        // Now open the file picker
+        const tmp = document.createElement('input');
+        tmp.type = 'file';
+        tmp.accept = 'image/*';
+        tmp.style.position = 'absolute';
+        tmp.style.left = '-9999px';
+        document.body.appendChild(tmp);
+
+        tmp.addEventListener('change', async () => {
+          const files = tmp.files;
+          if (!files || files.length === 0) {
+            uploadStatus.textContent = 'No file selected';
+            document.body.removeChild(tmp);
+            return;
+          }
+          switchToOcrQaMode();
+          uploadStatus.textContent = 'Uploading...';
+          uploadStatus.className = 'status-text text-warning';
+          const fd = new FormData();
+          fd.append('file', files[0]);
+          try {
+            const r = await fetch('/upload-image', { method: 'POST', body: fd });
+            if (!r.ok) {
+              const j = await r.json().catch(()=>({}));
+              uploadStatus.textContent = 'Upload failed: ' + (j.detail || r.statusText);
+              uploadStatus.className = 'status-text text-error';
+              document.body.removeChild(tmp);
+              return;
+            }
+            const j = await r.json();
+            currentDocId = j.doc_id;
+            uploadStatus.textContent = `Uploaded (doc_id=${currentDocId})`;
+            uploadStatus.className = 'status-text text-success';
+          } catch (e) {
+            uploadStatus.textContent = 'Upload error: ' + String(e);
+            uploadStatus.className = 'status-text text-error';
+          }
+          document.body.removeChild(tmp);
+        });
+        tmp.click();
+      }, 350); // 350ms to allow UI to update and user to see the switch
+      return;
+    }
+    // Already in OCR Q&A mode: open file picker immediately
+    const tmp = document.createElement('input');
+    tmp.type = 'file';
+    tmp.accept = 'image/*';
+    tmp.style.position = 'absolute';
+    tmp.style.left = '-9999px';
+    document.body.appendChild(tmp);
+
+    tmp.addEventListener('change', async () => {
+      const files = tmp.files;
+      if (!files || files.length === 0) {
+        uploadStatus.textContent = 'No file selected';
+        uploadStatus.className = 'status-text';
+        document.body.removeChild(tmp);
+        return;
+      }
+      switchToOcrQaMode();
+      uploadStatus.textContent = 'Uploading...';
+      uploadStatus.className = 'status-text text-warning';
+      const fd = new FormData();
+      fd.append('file', files[0]);
+      try {
+        const r = await fetch('/upload-image', { method: 'POST', body: fd });
+        if (!r.ok) {
+          const j = await r.json().catch(()=>({}));
+          uploadStatus.textContent = 'Upload failed: ' + (j.detail || r.statusText);
+          uploadStatus.className = 'status-text text-error';
+          document.body.removeChild(tmp);
+          return;
+        }
+        const j = await r.json();
+        currentDocId = j.doc_id;
+        uploadStatus.textContent = `Uploaded (doc_id=${currentDocId})`;
+        uploadStatus.className = 'status-text text-success';
+      } catch (e) {
+        uploadStatus.textContent = 'Upload error: ' + String(e);
+        uploadStatus.className = 'status-text text-error';
+      }
+      document.body.removeChild(tmp);
+    });
+    tmp.click();
+  });
+}
+
+function setRunning(running){
+  if(running){
+    askBtnEl.disabled = true;
+    recalcBtn.disabled = true;
+    questionEl.disabled = true;
+    spinner.classList.remove('hidden');
+    spinnerText.classList.remove('hidden');
+    spinnerText.textContent = 'Processing...';
+    // hide the small status text while spinner is active
+    statusEl.textContent = '';
+    console.log('[DEBUG] Loading state: ON, spinner should be visible'); // Debug log
+  } else {
+    askBtnEl.disabled = false;
+    recalcBtn.disabled = false;
+    questionEl.disabled = false;
+    spinner.classList.add('hidden');
+    spinnerText.classList.add('hidden');
+    // restore default status
+    statusEl.textContent = 'Ready';
+    console.log('[DEBUG] Loading state: OFF, spinner should be hidden'); // Debug log
+  }
+}
+
+function renderPresenterTrace(text){
+  if(!text) return '—';
+  // Split traces into chunks separated by blank lines and render as collapsible
+  const parts = String(text).split(/\n\s*\n/).filter(Boolean);
+  const container = document.createElement('div');
+  parts.forEach((p, i) => {
+    const d = document.createElement('details');
+    const summary = document.createElement('summary');
+    summary.textContent = p.split('\n')[0].slice(0,80);
+    summary.style.cursor = 'pointer';
+    summary.style.padding = '0.5rem';
+    summary.style.borderRadius = '0.375rem';
+    summary.style.marginBottom = '0.5rem';
+    summary.style.backgroundColor = 'var(--bg-tertiary)';
+    const pre = document.createElement('pre');
+    pre.style.whiteSpace = 'pre-wrap';
+    pre.style.fontSize = '0.75rem';
+    pre.style.padding = '0.75rem';
+    pre.style.backgroundColor = 'var(--bg-secondary)';
+    pre.style.borderRadius = '0.375rem';
+    pre.style.margin = '0';
+    pre.textContent = p;
+    d.appendChild(summary);
+    d.appendChild(pre);
+    if(i===0) d.open = true;
+    container.appendChild(d);
+  });
+  return container;
+}
+
+// small helper to escape HTML
+const esc = s => String(s || '').replace(/&/g,'&amp;').replace(/</g,'&lt;').replace(/>/g,'&gt;');
+
+// Detect if text contains a pipe/markdown table and return true
+function isMarkdownTable(text){
+  if(!text) return false;
+  const lines = String(text).split(/\r?\n/).map(l=>l.trim()).filter(Boolean);
+  if(lines.length < 2) return false;
+  // Count lines that contain pipe separators
+  const pipeLines = lines.filter(l => l.includes('|'));
+  if(pipeLines.length < 2) return false;
+  // If the second line is a markdown separator like | --- | --- | or ---|---
+  const second = lines[1];
+  if(/^\s*\|?\s*:?-{3,}:?\s*(\|\s*:?-{3,}:?\s*)+\|?\s*$/.test(second)) return true;
+  // Otherwise, if majority of non-empty lines contain pipes, treat as table
+  return (pipeLines.length / lines.length) >= 0.5;
+}
+
+// Parse a simple markdown/pipe table into a DOM <table>
+function parseMarkdownTable(text){
+  const lines = String(text).split(/\r?\n/).map(l=>l.trim()).filter(Boolean);
+  // find the first contiguous block of pipe lines
+  let start = -1, end = -1;
+  for(let i=0;i<lines.length;i++){
+    if(lines[i].includes('|')){ start = i; break; }
+  }
+  if(start === -1) return null;
+  for(let i=start;i<lines.length;i++){
+    if(!lines[i].includes('|')){ end = i; break; }
+  }
+  if(end === -1) end = lines.length;
+  const block = lines.slice(start, end);
+  const rows = block.map(r => {
+    let s = r;
+    if(s.startsWith('|')) s = s.slice(1);
+    if(s.endsWith('|')) s = s.slice(0,-1);
+    return s.split('|').map(c => c.trim());
+  });
+
+  // If second row is separator, treat first row as header
+  let header = null;
+  if(rows.length >= 2 && rows[1].every(c => /^:?-{3,}:?$/.test(c))) {
+    header = rows[0];
+    rows.splice(0,2);
+  }
+
+  const table = document.createElement('table');
+  table.className = 'answer-table';
+  const thead = document.createElement('thead');
+  const tbody = document.createElement('tbody');
+
+  if(header){
+    const tr = document.createElement('tr');
+    header.forEach(h => { const th = document.createElement('th'); th.textContent = h || ''; tr.appendChild(th); });
+    thead.appendChild(tr);
+  } else if(rows.length>0){
+    // use first row as header if no separator present
+    const tr = document.createElement('tr');
+    rows[0].forEach((_,i) => { const th = document.createElement('th'); th.textContent = `col ${i+1}`; tr.appendChild(th); });
+    thead.appendChild(tr);
+  }
+
+  rows.forEach(r => {
+    const tr = document.createElement('tr');
+    r.forEach(c => { const td = document.createElement('td'); td.textContent = c; tr.appendChild(td); });
+    tbody.appendChild(tr);
+  });
+
+  if(thead.childElementCount) table.appendChild(thead);
+  table.appendChild(tbody);
+  
+  // Create enhanced table wrapper with controls
+  const wrapper = document.createElement('div');
+  wrapper.className = 'table-wrapper';
+  
+  // Create table controls
+  const controls = document.createElement('div');
+  controls.className = 'table-controls';
+  
+  const controlsLeft = document.createElement('div');
+  controlsLeft.className = 'table-controls-left';
+  
+  const tableInfo = document.createElement('div');
+  tableInfo.className = 'table-info';
+  const rowCount = tbody.children.length;
+  const colCount = header ? header.length : (rows.length > 0 ? rows[0].length : 0);
+  tableInfo.textContent = `${rowCount} rows × ${colCount} columns`;
+  
+  controlsLeft.appendChild(tableInfo);
+  
+  const controlsRight = document.createElement('div');
+  controlsRight.className = 'table-controls-right';
+  
+  const resizeControls = document.createElement('div');
+  resizeControls.className = 'table-resize-controls';
+  
+  // Size control buttons
+  const sizes = [
+    { name: 'Compact', class: 'compact' },
+    { name: 'Comfortable', class: 'comfortable' },
+    { name: 'Spacious', class: 'spacious' }
+  ];
+  
+  sizes.forEach((size, index) => {
+    const btn = document.createElement('button');
+    btn.textContent = size.name;
+    btn.className = 'table-resize-btn';
+    if (index === 1) btn.classList.add('active'); // Default to comfortable
+    
+    btn.addEventListener('click', () => {
+      // Remove all size classes
+      table.classList.remove('compact', 'comfortable', 'spacious');
+      // Add selected size class
+      table.classList.add(size.class);
+      
+      // Update active button
+      resizeControls.querySelectorAll('.table-resize-btn').forEach(b => b.classList.remove('active'));
+      btn.classList.add('active');
+    });
+    
+    resizeControls.appendChild(btn);
+  });
+  
+  controlsRight.appendChild(resizeControls);
+  
+  controls.appendChild(controlsLeft);
+  controls.appendChild(controlsRight);
+  
+  // Create table container
+  const container = document.createElement('div');
+  container.className = 'table-container';
+  container.appendChild(table);
+  
+  wrapper.appendChild(controls);
+  wrapper.appendChild(container);
+  
+  // Set default size
+  table.classList.add('comfortable');
+  
+  return wrapper;
+}
+
+// heuristics: split logs into agent sections
+function splitLogs(logs) {
+  const sections = {memory: '', querycraft: '', presenter: ''};
+  if (!logs) {
+    console.log('[DEBUG] splitLogs: No logs provided');
+    return sections;
+  }
+  
+  console.log('[DEBUG] splitLogs: Input logs length:', logs.length);
+  console.log('[DEBUG] splitLogs: First 500 chars:', logs.substring(0, 500));
+  console.log('[DEBUG] splitLogs: Last 500 chars:', logs.substring(Math.max(0, logs.length - 500)));
+  
+  // Normalize line endings
+  const lines = logs.replace(/\r/g,'').split('\n');
+  console.log('[DEBUG] splitLogs: Total lines:', lines.length);
+
+  // Look for explicit markers first
+  let mode = null;
+  let memoryMarkerFound = false;
+  let querycraftMarkerFound = false;
+  let presenterMarkerFound = false;
+  
+  for (let i = 0; i < lines.length; i++) {
+    const line = lines[i];
+    const l = line.trim();
+    
+    // Match the exact agent section headers from agentic_workflow.py
+    if (/=== MEMORY AGENT ===/i.test(l) || /INITIAL MEMORY/i.test(l)) { 
+      mode = 'memory'; 
+      memoryMarkerFound = true;
+      console.log(`[DEBUG] splitLogs: Found MEMORY marker at line ${i}: "${l}"`);
+      continue; 
+    }
+    if (/=== QUERY CRAFT AGENT ===/i.test(l) || /QUERY_CRAFT/i.test(l) || /MICRO-HINT/i.test(l)) { 
+      mode = 'querycraft'; 
+      querycraftMarkerFound = true;
+      console.log(`[DEBUG] splitLogs: Found QUERYCRAFT marker at line ${i}: "${l}"`);
+      continue; 
+    }
+    if (/=== RESULT PRESENTER AGENT ===/i.test(l) || /PRESENTER/i.test(l)) { 
+      mode = 'presenter'; 
+      presenterMarkerFound = true;
+      console.log(`[DEBUG] splitLogs: Found PRESENTER marker at line ${i}: "${l}"`);
+      continue; 
+    }
+    // Additional patterns to catch query craft related logs
+    if (/\[FAST\]/i.test(l) || /\[GEN\]/i.test(l) || /\[MICRO-HINT\]/i.test(l)) { 
+      mode = 'querycraft'; 
+      console.log(`[DEBUG] splitLogs: Found QUERYCRAFT pattern at line ${i}: "${l}"`);
+    }
+    // Heuristic: SQL lines go to querycraft
+    if (/^(SELECT|WITH|INSERT|UPDATE|DELETE)\b/i.test(l)) { 
+      mode = 'querycraft'; 
+      sections.querycraft += l + '\n'; 
+      console.log(`[DEBUG] splitLogs: Found SQL pattern at line ${i}, mode set to querycraft`);
+      continue; 
+    }
+    if (!mode) {
+      // Seed with memory until a clearer marker appears
+      sections.memory += line + '\n';
+    } else {
+      sections[mode] += line + '\n';
+    }
+  }
+
+  console.log('[DEBUG] splitLogs: Markers found - Memory:', memoryMarkerFound, 'QueryCraft:', querycraftMarkerFound, 'Presenter:', presenterMarkerFound);
+  console.log('[DEBUG] splitLogs: Section lengths - Memory:', sections.memory.length, 'QueryCraft:', sections.querycraft.length, 'Presenter:', sections.presenter.length);
+  console.log('[DEBUG] splitLogs: Memory section preview:', sections.memory.substring(0, 200));
+  console.log('[DEBUG] splitLogs: QueryCraft section preview:', sections.querycraft.substring(0, 200));
+  console.log('[DEBUG] splitLogs: Presenter section preview:', sections.presenter.substring(0, 200));
+
+  return sections;
+}
+
+function findSQL(logs) {
+  if (!logs) return '';
+  // Primary: explicit markers
+  const markerRe = /<<<SQL_REVISED_START>>>([\s\S]*?)<<<SQL_REVISED_END>>>/m;
+  let m = logs.match(markerRe);
+  if (m) return m[1].trim();
+  const markerRe2 = /<<<SQL_START>>>([\s\S]*?)<<<SQL_END>>>/m;
+  m = logs.match(markerRe2);
+  if (m) return m[1].trim();
+  // Fallback: first standalone SQL-ish block up to semicolon
+  const fallback = logs.match(/^(?:.*?)(SELECT|WITH)[^;]{0,4000};/im);
+  if (fallback) {
+    const startIdx = fallback.index;
+    if (startIdx != null) {
+      const slice = logs.slice(startIdx).match(/([\s\S]*?;)/);
+      if (slice) return slice[1].trim();
+    }
+  }
+  return '';
+}
+
+async function runQuery(q){
+  if (!q) return alert('Please enter a question');
+  answerEl.textContent = 'Thinking...';
+  setRunning(true);
+  timeEl.textContent = '';
+  sqlEl.textContent = '-- SQL will appear here --';
+  memoryEl.textContent = querycraftEl.textContent = presenterEl.textContent = '—';
+  
+  const startTime = Date.now();
+  
+  try {
+    const body = { question: q, mode: currentMode };
+    if (currentMode === 'ocr_qa' && currentDocId) body.doc_id = currentDocId;
+    const res = await fetch('/api/query', {
+      method: 'POST', headers: { 'Content-Type': 'application/json' },
+      body: JSON.stringify(body)
+    });
+    let j = null;
+    if (!res.ok) {
+      // Read the body EXACTLY once, then parse — reading .json() then .text()
+      // on the same Response throws "body stream already read".
+      const raw = await res.text().catch(() => '');
+      try { j = raw ? JSON.parse(raw) : null; } catch (err) { j = null; }
+      if (!j) {
+        answerEl.textContent = 'Error ' + res.status + ': ' + (raw || res.statusText || 'request failed');
+        setRunning(false);
+        return;
+      }
+      if (j.error === 'worker_timeout') { answerEl.textContent = 'Worker timeout: ' + (j.message || 'The worker did not respond in time.'); setRunning(false); return; }
+      answerEl.textContent = j.error || j.message || JSON.stringify(j);
+      if (j.trace) presenterEl.textContent = j.trace;
+      setRunning(false);
+      return;
+    }
+    j = await res.json();
+    
+    const elapsed = ((Date.now() - startTime) / 1000).toFixed(1);
+    
+    if (j.answer) {
+      // Render markdown/pipe tables as HTML tables for readability
+      if(isMarkdownTable(j.answer)){
+        const node = parseMarkdownTable(j.answer);
+        answerEl.innerHTML = '';
+        if(node) answerEl.appendChild(node);
+        else answerEl.textContent = j.answer;
+      } else {
+        // Non-table answers: preserve plain text
+        answerEl.textContent = j.answer;
+      }
+    } else answerEl.textContent = JSON.stringify(j, null, 2);
+
+    // Chart contract: render ECharts visualization if the worker attached one
+    renderChart(j.chart);
+    // A query round-tripped: the empty-state hint is no longer needed
+    if (!j.chart) {
+      const pending = document.getElementById('chartPending');
+      if (pending) pending.classList.add('hidden');
+    }
+
+    // Prefer direct sql field from worker, else derive from logs
+    if (j.sql) {
+      const codeEl = document.getElementById('sql');
+      codeEl.textContent = j.sql;
+      if(window.Prism && Prism.highlightElement) Prism.highlightElement(codeEl);
+    }
+    
+    if (j.logs) {
+      console.log('[DEBUG] runQuery: Processing logs of length:', j.logs.length);
+      const sections = splitLogs(j.logs);
+      console.log('[DEBUG] runQuery: Sections received:', {
+        memory: sections.memory.length,
+        querycraft: sections.querycraft.length,
+        presenter: sections.presenter.length
+      });
+      
+      const memoryContent = sections.memory.trim() || '—';
+      const querycraftContent = sections.querycraft.trim() || '—';
+      const presenterContent = sections.presenter.trim() || '';
+      
+      console.log('[DEBUG] runQuery: Setting memory content length:', memoryContent.length);
+      console.log('[DEBUG] runQuery: Setting querycraft content length:', querycraftContent.length);
+      console.log('[DEBUG] runQuery: Setting presenter content length:', presenterContent.length);
+      
+      memoryEl.textContent = memoryContent;
+      querycraftEl.textContent = querycraftContent;
+      
+      if(presenterContent){ 
+        console.log('[DEBUG] runQuery: Rendering presenter trace');
+        const node = renderPresenterTrace(presenterContent); 
+        presenterEl.innerHTML = ''; 
+        presenterEl.appendChild(node); 
+      } else { 
+        console.log('[DEBUG] runQuery: No presenter content, setting dash');
+        presenterEl.textContent = '—'; 
+      }
+      if (!j.sql) {
+        const sql = findSQL(j.logs) || '';
+        if (sql) { 
+          const codeEl = document.getElementById('sql'); 
+          codeEl.textContent = sql; 
+          if(window.Prism && Prism.highlightElement) Prism.highlightElement(codeEl); 
+        }
+      }
+    } else {
+      console.log('[DEBUG] runQuery: No logs in response');
+    }
+    
+    statusEl.textContent = 'Done';
+    timeEl.textContent = `${elapsed}s`;
+    setRunning(false);
+  } catch (e) {
+    answerEl.textContent = 'Error: ' + String(e);
+    setRunning(false);
+  }
+}
+
+askBtn.onclick = () => runQuery(questionEl.value.trim());
+
+recalcBtn.onclick = () => {
+  const q = questionEl.value.trim();
+  if (!q) return alert('Please enter a question');
+  const normalized = q.replace(/^(?:re-execute\s+)+/i, '');
+  const prefixed = 're-execute ' + normalized;
+  // do not modify the visible textarea; send the prefixed query directly
+  runQuery(prefixed);
+};
+
+// Enter key support
+questionEl.addEventListener('keydown', (e) => {
+  if (e.key === 'Enter' && (e.ctrlKey || e.metaKey)) {
+    e.preventDefault();
+    runQuery(questionEl.value.trim());
+  }
+});
+
+/* ============================================================================
+ * Phase 3/4 — chart rendering (ECharts, sepia palette)
+ * ========================================================================== */
+const SEPIA_PALETTE = [
+  '#C15F3C', '#A39B7C', '#8A6F4D', '#B98A63', '#7E8A5A',
+  '#C9A227', '#6E7B6F', '#A9603F', '#5E5A4E', '#D9B08C',
+];
+let chartInstance = null;
+
+function renderChart(spec) {
+  const box = document.getElementById('chartBox');
+  const canvas = document.getElementById('chartCanvas');
+  const caption = document.getElementById('chartCaption');
+  const pending = document.getElementById('chartPending');
+  if (!box || !canvas) return;
+  if (!spec || !spec.type) { box.classList.add('hidden'); if (chartInstance) { chartInstance.dispose(); chartInstance = null; } return; }
+  if (typeof echarts === 'undefined') { box.classList.add('hidden'); return; }
+  if (pending) pending.classList.add('hidden');
+
+  box.classList.remove('hidden');
+  if (chartInstance) chartInstance.dispose();
+  chartInstance = echarts.init(canvas);
+
+  const base = {
+    backgroundColor: 'transparent',
+    color: SEPIA_PALETTE,
+    tooltip: { trigger: spec.type === 'pie' ? 'item' : 'axis' },
+    grid: { left: 12, right: 18, top: 42, bottom: 12, containLabel: true },
+    textStyle: { color: '#5E5A4E', fontFamily: 'Inter, sans-serif' },
+  };
+
+  let option = base;
+  const xAxis = {
+    type: 'category',
+    data: spec.x || [],
+    axisLine: { lineStyle: { color: '#D6CFBF' } },
+    axisLabel: { color: '#5E5A4E', rotate: (spec.x || []).length > 8 ? 30 : 0, hideOverlap: true },
+  };
+  const yAxis = {
+    type: 'value',
+    splitLine: { lineStyle: { color: '#EDE9DF' } },
+    axisLabel: { color: '#8B8678' },
+  };
+  const seriesData = (spec.series || []).map(s => ({ name: s.name, values: s.values || [] }));
+
+  if (spec.type === 'pie') {
+    const data = (spec.x || []).map((lab, i) => ({
+      name: lab,
+      value: (seriesData[0] && seriesData[0].values[i]) || 0,
+    }));
+    option = {
+      ...base, tooltip: { trigger: 'item' },
+      legend: { bottom: 0, textStyle: { color: '#5E5A4E', fontSize: 11 }, type: 'scroll' },
+      series: [{
+        type: 'pie', radius: ['38%', '68%'], center: ['50%', '46%'],
+        data, itemStyle: { borderColor: '#FAF9F5', borderWidth: 2, borderRadius: 6 },
+        label: { color: '#5E5A4E', fontSize: 11 },
+      }],
+    };
+  } else {
+    const key = spec.type === 'line' ? 'line' : 'bar';
+    option = {
+      ...base, xAxis, yAxis,
+      legend: seriesData.length > 1 ? { top: 4, textStyle: { color: '#5E5A4E', fontSize: 11 } } : undefined,
+      series: seriesData.map(s => ({
+        type: key, name: s.name, data: s.values,
+        ...(key === 'line' ? { smooth: true, symbolSize: 5 } : { itemStyle: { borderRadius: [4, 4, 0, 0] } }),
+      })),
+    };
+  }
+
+  chartInstance.setOption(option);
+  if (caption) {
+    caption.textContent = (spec.title || '') + (spec.rows_in_result ? ` · ${spec.rows_in_result} data points` : '');
+  }
+}
+
+window.addEventListener('resize', () => { if (chartInstance) chartInstance.resize(); });
+
+/* ============================================================================
+ * Dynamic database profile manager (Phase 1 API + Phase 4 UI)
+ * ========================================================================== */
+const schemaProfilesEl = document.getElementById('schemaProfiles');
+const schemaTablesEl = document.getElementById('schemaTables');
+const schemaNameEl = document.getElementById('schemaName');
+const schemaUrlEl = document.getElementById('schemaUrl');
+const schemaAddBtn = document.getElementById('schemaAddBtn');
+const schemaFileBtn = document.getElementById('schemaFileBtn');
+const dbBadge = document.getElementById('dbBadge');
+const dbTag = document.getElementById('dbTag');
+
+async function apiSchemas(method, path, body, isForm) {
+  const opts = { method, headers: {} };
+  if (body !== undefined) {
+    if (isForm) { opts.body = body; }
+    else { opts.headers['Content-Type'] = 'application/json'; opts.body = JSON.stringify(body); }
+  }
+  const r = await fetch(path, opts);
+  let j = null;
+  try { j = await r.json(); } catch (e) { j = null; }
+  if (!r.ok) throw new Error((j && (j.error || j.detail)) || r.statusText);
+  return j;
+}
+
+function setActiveProfileName(name) {
+  if (dbBadge) dbBadge.textContent = name || '—';
+  if (dbTag) dbTag.textContent = name || 'demo_employee';
+}
+
+async function refreshSchemaPanel() {
+  let data;
+  try { data = await apiSchemas('GET', '/api/schemas'); }
+  catch (e) {
+    if (schemaProfilesEl) schemaProfilesEl.innerHTML = '<div class="profile-item"><span class="profile-name">(schema service offline)</span></div>';
+    return;
+  }
+  const profiles = (data.profiles || []);
+  setActiveProfileName((profiles.find(p => p.active) || {}).name || '—');
+
+  if (schemaProfilesEl) {
+    schemaProfilesEl.innerHTML = '';
+    if (!profiles.length) {
+      schemaProfilesEl.innerHTML = '<div class="profile-item"><span class="profile-name">No profiles yet — add one below</span></div>';
+    }
+    for (const p of profiles) {
+      const item = document.createElement('div');
+      item.className = 'profile-item' + (p.active ? ' active' : '');
+      const dot = document.createElement('span');
+      dot.className = 'profile-dot' + (p.active ? ' on' : '');
+      const nm = document.createElement('span');
+      nm.className = 'profile-name'; nm.textContent = p.name; nm.title = p.db_url || '';
+      const kind = document.createElement('span');
+      kind.className = 'profile-kind'; kind.textContent = p.kind || 'user';
+      const actions = document.createElement('span'); actions.className = 'profile-actions';
+      if (!p.active) {
+        const sw = document.createElement('button');
+        sw.className = 'profile-btn'; sw.textContent = 'Switch';
+        sw.onclick = async () => {
+          sw.disabled = true; sw.textContent = '…';
+          try { await apiSchemas('POST', '/api/schemas/switch/' + p.id); await refreshSchemaPanel(); }
+          catch (e) { alert('Switch failed: ' + e.message); sw.disabled = false; sw.textContent = 'Switch'; }
+        };
+        actions.appendChild(sw);
+      }
+      const del = document.createElement('button');
+      del.className = 'profile-btn danger'; del.textContent = '×';
+      del.title = 'Remove profile';
+      del.onclick = async () => {
+        if (!confirm(`Remove profile "${p.name}"?`)) return;
+        try { await apiSchemas('DELETE', '/api/schemas/' + p.id); await refreshSchemaPanel(); }
+        catch (e) { alert('Delete failed: ' + e.message); }
+      };
+      actions.appendChild(del);
+      item.append(dot, nm, kind, actions);
+      schemaProfilesEl.appendChild(item);
+    }
+  }
+
+  // Current schema tables (best-effort)
+  try {
+    const cur = await apiSchemas('GET', '/api/schema/current');
+    if (schemaTablesEl) {
+      schemaTablesEl.innerHTML = '';
+      const tables = cur.tables || [];
+      if (!tables.length) { schemaTablesEl.innerHTML = ''; return; }
+      for (const t of tables.slice(0, 10)) {
+        const d = document.createElement('div');
+        d.className = 'tbl';
+        d.innerHTML = `<b>${escapeHtml(t.name)}</b> <span class="rc">(${t.row_count ?? '?'} rows · ${t.columns.length} cols)</span>`;
+        d.title = t.columns.join(', ');
+        schemaTablesEl.appendChild(d);
+      }
+    }
+  } catch (e) { /* schema detail endpoint optional */ }
+}
+
+function escapeHtml(s) {
+  return String(s).replace(/[&<>"']/g, c => ({ '&': '&amp;', '<': '&lt;', '>': '&gt;', '"': '&quot;', "'": '&#39;' }[c]));
+}
+
+async function addSchemaFromUrl() {
+  const name = (schemaNameEl && schemaNameEl.value.trim()) || '';
+  const url = (schemaUrlEl && schemaUrlEl.value.trim()) || '';
+  if (!name || !url) { alert('Provide both a profile name and a connection URL (or use file upload).'); return; }
+  schemaAddBtn.disabled = true; schemaAddBtn.textContent = 'Connecting…';
+  try {
+    await apiSchemas('POST', '/api/schemas/add', { name, connection_url: url });
+    schemaNameEl.value = ''; schemaUrlEl.value = '';
+    await refreshSchemaPanel();
+    statusEl.textContent = `Connected to "${name}"`;
+  } catch (e) {
+    alert('Connect failed: ' + e.message);
+  } finally {
+    schemaAddBtn.disabled = false; schemaAddBtn.textContent = 'Connect';
+  }
+}
+
+function addSchemaFromFile() {
+  const tmp = document.createElement('input');
+  tmp.type = 'file';
+  tmp.accept = '.sqlite,.db,.sqlite3,.csv';
+  tmp.style.position = 'absolute'; tmp.style.left = '-9999px';
+  document.body.appendChild(tmp);
+  tmp.addEventListener('change', async () => {
+    const f = tmp.files && tmp.files[0];
+    document.body.removeChild(tmp);
+    if (!f) return;
+    if (!schemaNameEl.value.trim()) schemaNameEl.value = f.name.replace(/\.[^.]+$/, '');
+    const fd = new FormData();
+    fd.append('file', f);
+    fd.append('name', schemaNameEl.value.trim());
+    fd.append('activate', 'true');
+    schemaFileBtn.disabled = true; schemaFileBtn.textContent = 'Importing…';
+    try {
+      await fetch('/api/schemas/add-file', { method: 'POST', body: fd }).then(async r => {
+        if (!r.ok) { const j = await r.json().catch(() => ({})); throw new Error(j.detail || r.statusText); }
+        return r.json();
+      });
+      await refreshSchemaPanel();
+      statusEl.textContent = `Imported "${schemaNameEl.value.trim()}"`;
+      schemaNameEl.value = '';
+    } catch (e) {
+      alert('Import failed: ' + e.message);
+    } finally {
+      schemaFileBtn.disabled = false; schemaFileBtn.textContent = 'Upload .sqlite / .csv';
+    }
+  });
+  tmp.click();
+}
+
+if (schemaAddBtn) schemaAddBtn.onclick = addSchemaFromUrl;
+if (schemaFileBtn) schemaFileBtn.onclick = addSchemaFromFile;
+
+// Initial load
+refreshSchemaPanel();
+

web/static/index.html

@@ -0,0 +1,690 @@
+<!doctype html>
+<html lang="en">
+
+<head>
+    <meta charset="utf-8" />
+    <meta name="viewport" content="width=device-width, initial-scale=1" />
+    <title>QueryLens — Ask your data anything</title>
+    <link rel="icon"
+        href="data:image/svg+xml;base64,PHN2ZyB3aWR0aD0iMjQiIGhlaWdodD0iMjQiIHZpZXdCb3g9IjAgMCAyNCAyNCIgZmlsbD0ibm9uZSIgeG1sbnM9Imh0dHA6Ly93d3cudzMub3JnLzIwMDAvc3ZnIj4KPGNpcmNsZSBjeD0iMTIiIGN5PSIxMiIgcj0iMTAiIGZpbGw9IiNDMTVGM0MiLz4KPHN2ZyB3aWR0aD0iMTYiIGhlaWdodD0iMTYiIHZpZXdCb3g9IjAgMCAyNCAyNCIgZmlsbD0ibm9uZSIgeD0iNCIgeT0iNCBjbGFzcz0icXVlcnktaWNvbiI+CjxwYXRoIGQ9Ik0xMiAyVjdNMTIgN0wxNSAzTTUgOS41SDkuNU05LjUgOS41TDYgNk0xOSAxNC41SDE0LjVNMTQuNSAxNC41TDE4IDE4TTEyIDE3VjIyTTEyIDIyTDkgMTkiIHN0cm9rZT0id2hpdGUiIHN0cm9rZS13aWR0aD0iMiIgc3Ryb2tlLWxpbmVjYXA9InJvdW5kIiBzdHJva2UtbGluZWpvaW49InJvdW5kIi8+Cjwvc3ZnPjwvc3ZnPg==">
+
+    <!-- Fonts -->
+    <link rel="preconnect" href="https://fonts.googleapis.com">
+    <link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
+    <link
+        href="https://fonts.googleapis.com/css2?family=Lora:ital,wght@0,400;0,500;0,600;1,400&family=Inter:wght@300;400;500;600;700&display=swap"
+        rel="stylesheet">
+
+    <style>
+        :root {
+            /* ── Sepia / coffee palette (Claude-style) ─────────────────── */
+            --clay: #C15F3C;              /* terracotta primary */
+            --clay-dark: #A84F2F;
+            --clay-soft: #E9C7B8;
+            --ocean: #5E6FA3;             /* muted blue for info */
+
+            --bg-primary: #FAF9F5;        /* ivory */
+            --bg-secondary: #F5F1EA;      /* warm sand */
+            --bg-tertiary: #EFEAE0;
+            --card-bg: #FFFFFF;
+            --code-bg: #F0EEE6;
+
+            --border: #E5DFD3;
+            --border-light: #EDE9DF;
+            --border-strong: #D6CFBF;
+
+            --text-primary: #29261B;      /* dark olive-brown */
+            --text-secondary: #5E5A4E;
+            --text-tertiary: #8B8678;
+            --text-on-clay: #FDF9F4;
+
+            --success: #6E7B4F;
+            --warning: #B98A23;
+            --error: #B04A32;
+
+            /* Spacing */
+            --space-1: 0.25rem;
+            --space-2: 0.5rem;
+            --space-3: 0.75rem;
+            --space-4: 1rem;
+            --space-5: 1.25rem;
+            --space-6: 1.5rem;
+            --space-8: 2rem;
+            --space-10: 2.5rem;
+
+            /* Shadows — warm and soft */
+            --shadow-sm: 0 1px 2px 0 rgb(74 60 35 / 0.05);
+            --shadow-md: 0 2px 6px -1px rgb(74 60 35 / 0.08), 0 1px 3px -1px rgb(74 60 35 / 0.06);
+            --shadow-lg: 0 8px 20px -6px rgb(74 60 35 / 0.12), 0 3px 8px -4px rgb(74 60 35 / 0.08);
+
+            --radius-sm: 6px;
+            --radius-md: 10px;
+            --radius-lg: 14px;
+            --radius-xl: 18px;
+
+            --transition-fast: 140ms ease;
+            --transition-normal: 240ms ease;
+        }
+
+        * , *::before, *::after { box-sizing: border-box; }
+
+        html, body {
+            height: 100%;
+            margin: 0;
+            font-family: 'Inter', -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, sans-serif;
+            background: var(--bg-primary);
+            color: var(--text-primary);
+            -webkit-font-smoothing: antialiased;
+            -moz-osx-font-smoothing: grayscale;
+            font-size: 15.5px;
+            line-height: 1.6;
+        }
+
+        h1, h2, h3, h4 { font-family: 'Lora', Georgia, serif; font-weight: 600; letter-spacing: -0.01em; }
+
+        ::selection { background: var(--clay-soft); color: var(--text-primary); }
+
+        /* ── Layout ──────────────────────────────────────────────────── */
+        .app-container {
+            min-height: 100vh;
+            display: flex;
+            flex-direction: column;
+        }
+
+        .header {
+            background: var(--bg-primary);
+            border-bottom: 1px solid var(--border-light);
+            padding: var(--space-5) var(--space-8);
+        }
+
+        .header-content {
+            max-width: 1240px;
+            margin: 0 auto;
+            display: flex;
+            align-items: center;
+            justify-content: space-between;
+            gap: var(--space-4);
+        }
+
+        .brand { display: flex; align-items: center; gap: 0.8rem; }
+
+        .brand-icon {
+            width: 40px; height: 40px;
+            border-radius: var(--radius-md);
+            background: var(--clay);
+            color: var(--text-on-clay);
+            display: grid; place-items: center;
+            box-shadow: var(--shadow-sm);
+        }
+
+        .brand-text h1 {
+            margin: 0; font-size: 1.32rem; line-height: 1.2; color: var(--text-primary);
+        }
+        .brand-text p {
+            margin: 1px 0 0; font-size: 0.82rem; color: var(--text-tertiary);
+        }
+
+        .header-tag {
+            font-size: 0.78rem;
+            color: var(--text-tertiary);
+            border: 1px solid var(--border);
+            background: var(--bg-secondary);
+            border-radius: 999px;
+            padding: 0.3rem 0.8rem;
+            white-space: nowrap;
+        }
+
+        .main-content {
+            max-width: 1240px;
+            width: 100%;
+            margin: 0 auto;
+            padding: var(--space-8) var(--space-8) var(--space-10);
+            flex: 1;
+        }
+
+        /* ── Query composer ──────────────────────────────────────────── */
+        .query-section { margin-bottom: var(--space-8); }
+
+        .query-body {
+            background: var(--card-bg);
+            border: 1px solid var(--border);
+            border-radius: var(--radius-lg);
+            box-shadow: var(--shadow-md);
+            padding: var(--space-5);
+            transition: border-color var(--transition-fast), box-shadow var(--transition-fast);
+        }
+        .query-body:focus-within {
+            border-color: var(--clay);
+            box-shadow: 0 0 0 3px rgb(193 95 60 / 0.10), var(--shadow-md);
+        }
+
+        .query-textarea {
+            width: 100%;
+            min-height: 96px;
+            border: none;
+            outline: none;
+            resize: vertical;
+            font-family: inherit;
+            font-size: 1.02rem;
+            color: var(--text-primary);
+            background: transparent;
+            line-height: 1.55;
+        }
+        .query-textarea::placeholder { color: var(--text-tertiary); }
+
+        .query-controls {
+            display: flex; align-items: center; gap: var(--space-3);
+            margin-top: var(--space-3);
+            padding-top: var(--space-3);
+            border-top: 1px solid var(--border-light);
+            flex-wrap: wrap;
+        }
+
+        .mode-select {
+            font-family: inherit;
+            font-size: 0.86rem;
+            color: var(--text-secondary);
+            background: var(--bg-secondary);
+            border: 1px solid var(--border);
+            border-radius: var(--radius-sm);
+            padding: 0.42rem 0.6rem;
+            cursor: pointer;
+        }
+
+        .btn {
+            display: inline-flex; align-items: center; gap: 0.45rem;
+            font-family: inherit; font-size: 0.9rem; font-weight: 600;
+            border-radius: var(--radius-sm);
+            padding: 0.5rem 1.05rem;
+            cursor: pointer;
+            border: 1px solid transparent;
+            transition: background var(--transition-fast), border-color var(--transition-fast),
+                        transform var(--transition-fast), opacity var(--transition-fast);
+        }
+        .btn:active { transform: translateY(1px); }
+        .btn:disabled { opacity: 0.55; cursor: not-allowed; }
+
+        .btn-primary {
+            background: var(--clay);
+            color: var(--text-on-clay);
+            border-color: var(--clay);
+        }
+        .btn-primary:hover:not(:disabled) { background: var(--clay-dark); border-color: var(--clay-dark); }
+
+        .btn-secondary {
+            background: var(--card-bg);
+            color: var(--text-secondary);
+            border-color: var(--border-strong);
+        }
+        .btn-secondary:hover:not(:disabled) {
+            background: var(--bg-secondary);
+            color: var(--text-primary);
+        }
+
+        .status-area {
+            margin-left: auto;
+            display: flex; align-items: center; gap: 0.8rem;
+            font-size: 0.84rem; color: var(--text-tertiary);
+        }
+
+        .spinner {
+            width: 16px; height: 16px;
+            border: 2px solid var(--clay-soft);
+            border-top-color: var(--clay);
+            border-radius: 50%;
+            animation: spin 0.8s linear infinite;
+        }
+        @keyframes spin { to { transform: rotate(360deg); } }
+
+        .hidden { display: none !important; }
+
+        /* ── Results grid ────────────────────────────────────────────── */
+        .results-grid {
+            display: grid;
+            grid-template-columns: minmax(0, 1fr) 340px;
+            gap: var(--space-6);
+            align-items: start;
+        }
+        @media (max-width: 1020px) {
+            .results-grid { grid-template-columns: 1fr; }
+        }
+
+        .card {
+            background: var(--card-bg);
+            border: 1px solid var(--border);
+            border-radius: var(--radius-lg);
+            box-shadow: var(--shadow-sm);
+            overflow: hidden;
+        }
+        .card + .card { margin-top: var(--space-6); }
+
+        .card-header {
+            display: flex; align-items: center; gap: 0.5rem;
+            padding: 0.95rem var(--space-5);
+            border-bottom: 1px solid var(--border-light);
+            background: transparent;
+        }
+        .card-title {
+            margin: 0; font-size: 0.98rem; color: var(--text-primary);
+            display: flex; align-items: center; gap: 0.5rem;
+            font-family: 'Lora', Georgia, serif;
+        }
+        .card-title svg { color: var(--clay); }
+        .card-content { padding: var(--space-5); }
+
+        .answer-content { white-space: normal; color: var(--text-primary); }
+
+        /* ── Chart box (Phase 3 contract) ───────────────────────────── */
+        .chart-box {
+            margin-top: var(--space-4);
+            border: 1px solid var(--border-light);
+            background: var(--bg-secondary);
+            border-radius: var(--radius-md);
+            padding: var(--space-3);
+        }
+        .chart-pending {
+            margin-top: var(--space-4);
+            border: 1px dashed var(--border-light);
+            background: var(--bg-secondary);
+            border-radius: var(--radius-md);
+            padding: var(--space-4) var(--space-3);
+            font-size: 0.82rem;
+            color: var(--text-tertiary);
+            line-height: 1.6;
+        }
+        .chart-pending.hidden { display: none; }
+        #chartCanvas { width: 100%; height: 340px; }
+        .chart-caption {
+            margin: 0.4rem 0.2rem 0;
+            font-size: 0.78rem; color: var(--text-tertiary);
+            text-align: right;
+        }
+
+        /* ── SQL block ──────────────────────────────────────────────── */
+        details summary { outline: none; }
+        .sql-content {
+            margin: 0;
+            background: var(--code-bg);
+            border: 1px solid var(--border-light);
+            border-radius: var(--radius-md);
+            padding: var(--space-4);
+            overflow-x: auto;
+            font-size: 0.86rem;
+            line-height: 1.55;
+        }
+        code.language-sql, pre.sql-content code {
+            font-family: 'Cascadia Code', 'Fira Code', Consolas, monospace;
+            color: #4A4234;
+        }
+
+        /* ── Agent panels sidebar ───────────────────────────────────── */
+        .agent-panels { display: flex; flex-direction: column; gap: var(--space-4); }
+
+        .panel {
+            background: var(--card-bg);
+            border: 1px solid var(--border);
+            border-radius: var(--radius-lg);
+            box-shadow: var(--shadow-sm);
+            overflow: hidden;
+        }
+        .panel summary { list-style: none; }
+        .panel summary::-webkit-details-marker { display: none; }
+
+        .panel-header {
+            display: flex; align-items: center; justify-content: space-between;
+            padding: 0.8rem var(--space-4);
+            cursor: pointer;
+            user-select: none;
+        }
+        .panel-header:hover { background: var(--bg-secondary); }
+        .panel-title {
+            display: flex; align-items: center; gap: 0.5rem;
+            font-size: 0.84rem; font-weight: 600; color: var(--text-primary);
+            font-family: 'Lora', Georgia, serif;
+        }
+        .panel-title svg { color: var(--clay); }
+
+        .panel-content {
+            padding: 0 var(--space-4) var(--space-4);
+            font-size: 0.78rem;
+            color: var(--text-secondary);
+            max-height: 260px;
+            overflow-y: auto;
+            white-space: pre-wrap;
+            word-break: break-word;
+            font-family: Consolas, monospace;
+        }
+
+        /* ── Database panel ─────────────────────────────────────────── */
+        .db-badge {
+            margin-left: auto;
+            font-size: 0.7rem; font-weight: 600;
+            color: var(--clay);
+            background: rgb(193 95 60 / 0.10);
+            border: 1px solid rgb(193 95 60 / 0.25);
+            border-radius: 999px;
+            padding: 0.12rem 0.55rem;
+        }
+
+        .profile-list { display: flex; flex-direction: column; gap: 6px; padding: 0 var(--space-4) var(--space-3); }
+        .profile-item {
+            display: flex; align-items: center; gap: 0.5rem;
+            border: 1px solid var(--border);
+            background: var(--bg-secondary);
+            border-radius: var(--radius-sm);
+            padding: 0.5rem 0.65rem;
+            font-size: 0.8rem;
+        }
+        .profile-item.active { border-color: var(--clay); background: rgb(193 95 60 / 0.06); }
+        .profile-name { font-weight: 600; color: var(--text-primary); overflow: hidden; text-overflow: ellipsis; white-space: nowrap; }
+        .profile-kind { font-size: 0.68rem; color: var(--text-tertiary); text-transform: uppercase; letter-spacing: 0.04em; }
+        .profile-actions { margin-left: auto; display: flex; gap: 4px; }
+        .profile-btn {
+            border: 1px solid var(--border-strong);
+            background: var(--card-bg);
+            color: var(--text-secondary);
+            font-size: 0.7rem; font-weight: 600;
+            border-radius: 4px; padding: 2px 8px; cursor: pointer;
+            font-family: inherit;
+        }
+        .profile-btn:hover { border-color: var(--clay); color: var(--clay); }
+        .profile-btn.danger:hover { border-color: var(--error); color: var(--error); }
+        .profile-dot {
+            width: 7px; height: 7px; border-radius: 50%;
+            background: var(--border-strong); flex-shrink: 0;
+        }
+        .profile-dot.on { background: var(--success); }
+
+        .schema-add { padding: 0 var(--space-4) var(--space-4); border-top: 1px dashed var(--border); padding-top: var(--space-3); }
+        .schema-add label {
+            display: block; font-size: 0.72rem; font-weight: 600;
+            color: var(--text-tertiary); margin: 0.5rem 0 3px;
+            text-transform: uppercase; letter-spacing: 0.05em;
+        }
+        .schema-add input[type="text"] {
+            width: 100%; font-family: inherit; font-size: 0.8rem;
+            border: 1px solid var(--border); border-radius: var(--radius-sm);
+            padding: 0.4rem 0.55rem; color: var(--text-primary);
+            background: var(--card-bg);
+        }
+        .schema-add input[type="text"]:focus { outline: none; border-color: var(--clay); }
+        .schema-add .row2 { display: flex; gap: 6px; margin-top: 0.55rem; }
+        .schema-add .row2 .btn { flex: 1; justify-content: center; padding: 0.42rem 0.5rem; font-size: 0.78rem; }
+
+        .schema-tables { padding: 0 var(--space-4) var(--space-4); }
+        .schema-tables .tbl {
+            font-size: 0.75rem; color: var(--text-secondary);
+            border-bottom: 1px dotted var(--border);
+            padding: 3px 0;
+        }
+        .schema-tables .tbl b { color: var(--text-primary); font-weight: 600; }
+        .schema-tables .tbl .rc { color: var(--text-tertiary); font-size: 0.68rem; }
+
+        /* ── Footer ─────────────────────────────────────────────────── */
+        .footer {
+            text-align: center;
+            padding: var(--space-6);
+            color: var(--text-tertiary);
+            font-size: 0.8rem;
+            border-top: 1px solid var(--border-light);
+        }
+        .footer p { margin: 0; }
+
+        /* Markdown tables inside answer */
+        .answer-content table {
+            border-collapse: collapse;
+            width: 100%;
+            margin: 0.6rem 0;
+            font-size: 0.86rem;
+        }
+        .answer-content th, .answer-content td {
+            border: 1px solid var(--border);
+            padding: 0.4rem 0.7rem;
+            text-align: left;
+        }
+        .answer-content th { background: var(--bg-secondary); font-weight: 600; }
+        .answer-content tr:nth-child(even) td { background: var(--bg-secondary); }
+    </style>
+</head>
+
+<body>
+    <div class="app-container">
+        <!-- Header -->
+        <header class="header">
+            <div class="header-content">
+                <div class="brand">
+                    <div class="brand-icon">
+                        <svg width="22" height="22" viewBox="0 0 24 24" fill="none" xmlns="http://www.w3.org/2000/svg">
+                            <path d="M12 2V7M12 7L15 3M5 9.5H9.5M9.5 9.5L6 6M19 14.5H14.5M14.5 14.5L18 18M12 17V22M12 22L9 19"
+                                stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" />
+                        </svg>
+                    </div>
+                    <div class="brand-text">
+                        <h1>QueryLens</h1>
+                        <p>Ask your data anything.</p>
+                    </div>
+                </div>
+                <span class="header-tag" id="dbTag">demo_employee</span>
+            </div>
+        </header>
+
+        <!-- Main Content -->
+        <main class="main-content">
+            <!-- Query Section -->
+            <section class="query-section">
+                <div class="query-body">
+                    <textarea id="question" class="query-textarea"
+                        placeholder="Ask a question about your data — e.g. “Which department has the highest average salary?” or “Show me sales by month as a chart.”"></textarea>
+
+                    <div class="query-controls">
+                        <select id="modeToggle" class="mode-select">
+                            <option value="text2sql">Text-to-SQL</option>
+                            <option value="ocr_qa">Document Q&amp;A</option>
+                        </select>
+
+                        <button id="ask" class="btn btn-primary">
+                            <svg width="15" height="15" viewBox="0 0 24 24" fill="none" stroke="currentColor"
+                                stroke-width="2">
+                                <path d="M12 19l7-7 3 3-7 7-3-3z" />
+                                <path d="M18 13l-1.5-7.5L2 2l3.5 14.5L13 18l5-5z" />
+                            </svg>
+                            Ask
+                        </button>
+
+                        <button id="recalc" class="btn btn-secondary">
+                            <svg width="15" height="15" viewBox="0 0 24 24" fill="none" stroke="currentColor"
+                                stroke-width="2">
+                                <path d="M1 4v6h6" />
+                                <path d="M23 20v-6h-6" />
+                                <path
+                                    d="M20.49 9A9 9 0 0 0 5.64 5.64L1 10m22 4l-4.64 4.36A9 9 0 0 1 3.51 15" />
+                            </svg>
+                            Re-execute
+                        </button>
+
+                        <div id="uploadControls" class="hidden" style="display:flex; gap: 0.5rem; align-items: center;">
+                            <button id="uploadBtn" class="btn btn-secondary">
+                                <svg width="15" height="15" viewBox="0 0 24 24" fill="none" stroke="currentColor"
+                                    stroke-width="2">
+                                    <path d="M21 15v4a2 2 0 0 1-2 2H5a2 2 0 0 1-2-2v-4" />
+                                    <polyline points="17,8 12,3 7,8" />
+                                    <line x1="12" y1="3" x2="12" y2="15" />
+                                </svg>
+                                Upload Image
+                            </button>
+                            <span id="uploadStatus">No document</span>
+                        </div>
+
+                        <div class="status-area">
+                            <div id="spinner" class="spinner hidden"></div>
+                            <span id="spinnerText" class="hidden">Thinking…</span>
+                            <span id="status">Ready</span>
+                            <span id="time"></span>
+                        </div>
+                    </div>
+                </div>
+            </section>
+
+            <!-- Results Grid -->
+            <div class="results-grid">
+                <!-- Main Results Column -->
+                <div class="main-results">
+                    <!-- Answer Card -->
+                    <div class="card">
+                        <div class="card-header">
+                            <h3 class="card-title">
+                                <svg width="15" height="15" viewBox="0 0 24 24" fill="none" stroke="currentColor"
+                                    stroke-width="2">
+                                    <path d="M9 11l3 3L22 4" />
+                                    <path d="M21 12v7a2 2 0 0 1-2 2H5a2 2 0 0 1-2-2V5a2 2 0 0 1 2-2h11" />
+                                </svg>
+                                Answer
+                            </h3>
+                        </div>
+                        <div class="card-content">
+                            <div id="answer" class="answer-content">No answer yet — ask something above.</div>
+                            <div id="chartPending" class="chart-pending">
+                            📊 Charts &amp; analytics render here automatically when a successful query
+                            returns chart-friendly data — try “Show average salary by department”.
+                        </div>
+                        <div id="chartBox" class="chart-box hidden">
+                                <div id="chartCanvas"></div>
+                                <p class="chart-caption" id="chartCaption"></p>
+                            </div>
+                        </div>
+                    </div>
+
+                    <!-- SQL Card -->
+                    <div class="card">
+                        <details id="sql-details" open>
+                            <summary class="card-header">
+                                <h3 class="card-title">
+                                    <svg width="15" height="15" viewBox="0 0 24 24" fill="none" stroke="currentColor"
+                                        stroke-width="2">
+                                        <polyline points="16,18 22,12 16,6" />
+                                        <polyline points="8,6 2,12 8,18" />
+                                    </svg>
+                                    Generated SQL
+                                </h3>
+                            </summary>
+                            <div class="card-content">
+                                <pre
+                                    class="sql-content"><code id="sql" class="language-sql">-- SQL will appear here after the agent crafts a query --</code></pre>
+                            </div>
+                        </details>
+                    </div>
+                </div>
+
+                <!-- Sidebar -->
+                <div class="agent-panels">
+                    <!-- Database panel -->
+                    <div class="panel">
+                        <details open>
+                            <summary class="panel-header">
+                                <span class="panel-title">
+                                    <svg width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor"
+                                        stroke-width="2">
+                                        <ellipse cx="12" cy="5" rx="9" ry="3" />
+                                        <path d="M21 12c0 1.66-4 3-9 3s-9-1.34-9-3" />
+                                        <path d="M3 5v14c0 1.66 4 3 9 3s9-1.34 9-3V5" />
+                                    </svg>
+                                    Database
+                                    <span class="db-badge" id="dbBadge">—</span>
+                                </span>
+                            </summary>
+                            <div class="profile-list" id="schemaProfiles"></div>
+                            <div class="schema-add">
+                                <label for="schemaName">Add a database</label>
+                                <input type="text" id="schemaName" placeholder="Profile name (e.g. sales_2024)">
+                                <label for="schemaUrl" style="margin-top:0.6rem">Connection URL — or upload a file</label>
+                                <input type="text" id="schemaUrl"
+                                    placeholder="postgresql://user:pass@host/db  ·  sqlite:///path/file.db">
+                                <div class="row2">
+                                    <button id="schemaAddBtn" class="btn btn-primary">Connect</button>
+                                    <button id="schemaFileBtn" class="btn btn-secondary">Upload .sqlite / .csv</button>
+                                </div>
+                            </div>
+                            <div class="schema-tables" id="schemaTables"></div>
+                        </details>
+                    </div>
+
+                    <!-- Agent panels -->
+                    <div class="panel">
+                        <details open>
+                            <summary class="panel-header">
+                                <span class="panel-title">
+                                    <svg width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor"
+                                        stroke-width="2">
+                                        <path
+                                            d="M9 19c-5 1.5-5-2.5-7-3m14 6v-3.87a3.37 3.37 0 0 0-.94-2.61c3.14-.35 6.44-1.54 6.44-7A5.44 5.44 0 0 0 20 4.77 5.07 5.07 0 0 0 19.91 1S18.73.65 16 2.48a13.38 13.38 0 0 0-7 0C6.27.65 5.09 1 5.09 1A5.07 5.07 0 0 0 5 4.77a5.44 5.44 0 0 0-1.5 3.78c0 5.42 3.3 6.61 6.44 7A3.37 3.37 0 0 0 9 18.13V22" />
+                                    </svg>
+                                    Memory Agent
+                                </span>
+                                <svg width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor"
+                                    stroke-width="2">
+                                    <polyline points="6,9 12,15 18,9" />
+                                </svg>
+                            </summary>
+                            <div id="memory" class="panel-content">—</div>
+                        </details>
+                    </div>
+
+                    <div class="panel">
+                        <details open>
+                            <summary class="panel-header">
+                                <span class="panel-title">
+                                    <svg width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor"
+                                        stroke-width="2">
+                                        <path d="M12 20h9" />
+                                        <path d="M16.5 3.5a2.121 2.121 0 0 1 3 3L7 19l-4 1 1-4L16.5 3.5z" />
+                                    </svg>
+                                    Query Craft Agent
+                                </span>
+                                <svg width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor"
+                                    stroke-width="2">
+                                    <polyline points="6,9 12,15 18,9" />
+                                </svg>
+                            </summary>
+                            <div id="querycraft" class="panel-content">—</div>
+                        </details>
+                    </div>
+
+                    <div class="panel">
+                        <details open>
+                            <summary class="panel-header">
+                                <span class="panel-title">
+                                    <svg width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor"
+                                        stroke-width="2">
+                                        <path d="M14 2H6a2 2 0 0 0-2 2v16a2 2 0 0 0 2 2h12a2 2 0 0 0 2-2V8z" />
+                                        <polyline points="14,2 14,8 20,8" />
+                                        <line x1="16" y1="13" x2="8" y2="13" />
+                                        <line x1="16" y1="17" x2="8" y2="17" />
+                                        <polyline points="10,9 9,9 8,9" />
+                                    </svg>
+                                    Result Presenter Agent
+                                </span>
+                                <svg width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor"
+                                    stroke-width="2">
+                                    <polyline points="6,9 12,15 18,9" />
+                                </svg>
+                            </summary>
+                            <div id="presenter" class="panel-content">—</div>
+                        </details>
+                    </div>
+                </div>
+            </div>
+        </main>
+
+        <!-- Footer -->
+        <footer class="footer">
+            <p>QueryLens — multi-agent Text-to-SQL &amp; document intelligence</p>
+        </footer>
+    </div>
+
+    <script src="https://cdn.jsdelivr.net/npm/echarts@5.5.0/dist/echarts.min.js"></script>
+    <script src="/static/app.js"></script>
+    <script src="https://cdnjs.cloudflare.com/ajax/libs/prism/1.29.0/prism.min.js"></script>
+    <script src="https://cdnjs.cloudflare.com/ajax/libs/prism/1.29.0/components/prism-sql.min.js"></script>
+</body>
+
+</html>

web/static/table-demo.html

@@ -0,0 +1,338 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+    <meta charset="UTF-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+    <title>Table Demo</title>
+    <style>
+        /* Copy the same CSS variables and table styles from the main app */
+        :root {
+            --primary: #0f62fe;
+            --primary-hover: #0353e9;
+            --bg-primary: #FAFBFC;
+            --bg-secondary: #F4F6F8;
+            --bg-tertiary: #E8EDF3;
+            --card-bg: #ffffff;
+            --text-primary: #1A202C;
+            --text-secondary: #4A5568;
+            --text-tertiary: #718096;
+            --border: #D1D9E0;
+            --border-light: #E5EAF0;
+            --shadow-sm: 0 1px 2px 0 rgba(0, 0, 0, 0.05);
+            --shadow-lg: 0 10px 15px -3px rgba(0, 0, 0, 0.1), 0 4px 6px -2px rgba(0, 0, 0, 0.05);
+            --radius-sm: 0.25rem;
+            --radius-md: 0.375rem;
+            --radius-lg: 0.5rem;
+            --radius-xl: 0.75rem;
+            --space-1: 0.25rem;
+            --space-2: 0.5rem;
+            --space-3: 0.75rem;
+            --space-4: 1rem;
+            --space-6: 1.5rem;
+            --space-8: 2rem;
+            --space-12: 3rem;
+            --transition-normal: all 0.2s ease-in-out;
+        }
+
+        body {
+            font-family: 'Inter', -apple-system, BlinkMacSystemFont, sans-serif;
+            background: var(--bg-secondary);
+            color: var(--text-primary);
+            margin: 2rem;
+        }
+
+        .table-wrapper {
+            position: relative;
+            margin: var(--space-4) 0;
+            border-radius: var(--radius-lg);
+            box-shadow: var(--shadow-lg);
+            background: var(--card-bg);
+            border: 1px solid var(--border);
+            overflow: hidden;
+        }
+
+        .table-controls {
+            display: flex;
+            align-items: center;
+            justify-content: space-between;
+            padding: var(--space-3) var(--space-4);
+            background: var(--bg-tertiary);
+            border-bottom: 1px solid var(--border);
+            gap: var(--space-3);
+        }
+
+        .table-controls-left {
+            display: flex;
+            align-items: center;
+            gap: var(--space-3);
+        }
+
+        .table-controls-right {
+            display: flex;
+            align-items: center;
+            gap: var(--space-2);
+        }
+
+        .table-info {
+            font-size: 0.875rem;
+            color: var(--text-secondary);
+            font-weight: 500;
+        }
+
+        .table-resize-controls {
+            display: flex;
+            align-items: center;
+            gap: var(--space-2);
+        }
+
+        .table-resize-btn {
+            padding: var(--space-1) var(--space-2);
+            border: 1px solid var(--border);
+            border-radius: var(--radius-sm);
+            background: var(--bg-primary);
+            color: var(--text-secondary);
+            font-size: 0.75rem;
+            cursor: pointer;
+            transition: var(--transition-normal);
+        }
+
+        .table-resize-btn:hover {
+            background: var(--bg-secondary);
+            color: var(--text-primary);
+        }
+
+        .table-resize-btn.active {
+            background: var(--primary);
+            color: white;
+            border-color: var(--primary);
+        }
+
+        .table-container {
+            overflow: auto;
+            max-height: 70vh;
+            position: relative;
+        }
+
+        .answer-table {
+            width: 100%;
+            border-collapse: separate;
+            border-spacing: 0;
+            font-family: 'Inter', -apple-system, BlinkMacSystemFont, sans-serif;
+            font-size: 0.875rem;
+            background: var(--card-bg);
+            table-layout: auto;
+            min-width: 100%;
+        }
+
+        .answer-table.compact {
+            table-layout: fixed;
+        }
+
+        .answer-table.comfortable {
+            table-layout: auto;
+        }
+
+        .answer-table.spacious {
+            table-layout: auto;
+            font-size: 1rem;
+        }
+
+        .answer-table th,
+        .answer-table td {
+            padding: var(--space-3) var(--space-4);
+            text-align: left;
+            border-bottom: 1px solid var(--border-light);
+            border-right: 1px solid var(--border-light);
+            vertical-align: top;
+            position: relative;
+        }
+
+        .answer-table.compact th,
+        .answer-table.compact td {
+            padding: var(--space-2) var(--space-3);
+            max-width: 120px;
+            overflow: hidden;
+            text-overflow: ellipsis;
+            white-space: nowrap;
+        }
+
+        .answer-table.comfortable th,
+        .answer-table.comfortable td {
+            padding: var(--space-3) var(--space-4);
+            max-width: 200px;
+            word-wrap: break-word;
+            white-space: normal;
+        }
+
+        .answer-table.spacious th,
+        .answer-table.spacious td {
+            padding: var(--space-4) var(--space-6);
+            max-width: none;
+            word-wrap: break-word;
+            white-space: normal;
+        }
+
+        .answer-table th {
+            background: #F1F5F9;
+            font-weight: 600;
+            color: #1A202C;
+            position: sticky;
+            top: 0;
+            z-index: 10;
+            border-top: none;
+            border-bottom: 2px solid #D1D9E0;
+        }
+
+        .answer-table th:last-child,
+        .answer-table td:last-child {
+            border-right: none;
+        }
+
+        .answer-table tbody tr:hover {
+            background: #E8F4FD;
+        }
+
+        .answer-table tbody tr:nth-child(even) {
+            background: #F7F9FC;
+        }
+
+        .answer-table tbody tr:nth-child(even):hover {
+            background: #E8F4FD;
+        }
+    </style>
+</head>
+<body>
+    <h1>Enhanced Table Demo</h1>
+    <p>This demonstrates the new resizable table interface:</p>
+    
+    <div class="table-wrapper">
+        <div class="table-controls">
+            <div class="table-controls-left">
+                <div class="table-info">9 rows × 7 columns</div>
+            </div>
+            <div class="table-controls-right">
+                <div class="table-resize-controls">
+                    <button class="table-resize-btn" onclick="setTableSize('compact')">Compact</button>
+                    <button class="table-resize-btn active" onclick="setTableSize('comfortable')">Comfortable</button>
+                    <button class="table-resize-btn" onclick="setTableSize('spacious')">Spacious</button>
+                </div>
+            </div>
+        </div>
+        <div class="table-container">
+            <table class="answer-table comfortable">
+                <thead>
+                    <tr>
+                        <th>dept_name</th>
+                        <th>avg_salary</th>
+                        <th>num_employee</th>
+                        <th>male_salary</th>
+                        <th>female_salary</th>
+                        <th>avg_male_salary</th>
+                        <th>avg_female_salary</th>
+                    </tr>
+                </thead>
+                <tbody>
+                    <tr>
+                        <td>Sales</td>
+                        <td>$86,123.43</td>
+                        <td>130</td>
+                        <td>$6,181,954</td>
+                        <td>$5,014,084</td>
+                        <td>$84,684.37</td>
+                        <td>$87,966.44</td>
+                    </tr>
+                    <tr>
+                        <td>Marketing</td>
+                        <td>$81,874.03</td>
+                        <td>34</td>
+                        <td>$1,736,164</td>
+                        <td>$1,047,554</td>
+                        <td>$82,674.38</td>
+                        <td>$80,581.15</td>
+                    </tr>
+                    <tr>
+                        <td>Finance</td>
+                        <td>$76,399.98</td>
+                        <td>43</td>
+                        <td>$2,026,534</td>
+                        <td>$1,258,664</td>
+                        <td>$75,056.70</td>
+                        <td>$78,666.75</td>
+                    </tr>
+                    <tr>
+                        <td>Development</td>
+                        <td>$69,564.44</td>
+                        <td>203</td>
+                        <td>$8,159,084</td>
+                        <td>$5,962,504</td>
+                        <td>$69,735.73</td>
+                        <td>$69,331.42</td>
+                    </tr>
+                    <tr>
+                        <td>Research</td>
+                        <td>$69,147.39</td>
+                        <td>51</td>
+                        <td>$2,279,864</td>
+                        <td>$1,246,654</td>
+                        <td>$69,086.76</td>
+                        <td>$69,258.56</td>
+                    </tr>
+                    <tr>
+                        <td>Production</td>
+                        <td>$68,537.28</td>
+                        <td>186</td>
+                        <td>$7,929,244</td>
+                        <td>$4,818,694</td>
+                        <td>$68,355.55</td>
+                        <td>$68,838.43</td>
+                    </tr>
+                    <tr>
+                        <td>Quality Management</td>
+                        <td>$64,859.40</td>
+                        <td>48</td>
+                        <td>$2,112,644</td>
+                        <td>$1,000,604</td>
+                        <td>$64,019.61</td>
+                        <td>$66,706.93</td>
+                    </tr>
+                    <tr>
+                        <td>Customer Service</td>
+                        <td>$63,643.00</td>
+                        <td>61</td>
+                        <td>$1,730,834</td>
+                        <td>$2,151,394</td>
+                        <td>$61,815.43</td>
+                        <td>$65,193.67</td>
+                    </tr>
+                    <tr>
+                        <td>Human Resources</td>
+                        <td>$63,507.28</td>
+                        <td>46</td>
+                        <td>$2,151,834</td>
+                        <td>$769,499</td>
+                        <td>$63,289.29</td>
+                        <td>$64,124.92</td>
+                    </tr>
+                </tbody>
+            </table>
+        </div>
+    </div>
+
+    <script>
+        function setTableSize(size) {
+            const table = document.querySelector('.answer-table');
+            const buttons = document.querySelectorAll('.table-resize-btn');
+            
+            // Remove all size classes
+            table.classList.remove('compact', 'comfortable', 'spacious');
+            
+            // Add selected size class
+            table.classList.add(size);
+            
+            // Update active button
+            buttons.forEach(btn => btn.classList.remove('active'));
+            event.target.classList.add('active');
+        }
+    </script>
+</body>
+</html>