fix: Round 14 — 28 Tier 1 + Tier 2 bugs fixed across 17 source files

Anti-hallucination integrity (T1):
- #100: Detect verification LLM system failure (all calls fail → refuse)
- #101: Fix validate_citations SQL/web source matching (dead code → working)
- #102: Citation count uses chunks (citable units) not unique files
- #103: Clamp max_iterations to min 1 (can't bypass via iterations=0)

Pre-release quality (T2 HIGH):
- #104: Add LIKE ESCAPE in all SQL fallback paths
- #105: Gemini safety block returns "" (was bypassing stack)
- #106: Cap reasoning model budget at 128000
- #107: Handle non-numeric temperature/max_tokens gracefully
- #108: Fix clarification history duplication

Pre-release quality (T2 MEDIUM):
- #109: Anthropic temperature clamp to 1.0
- #110: SQL query 10-second timeout via progress handler
- #111: Rich markup escape for user/LLM content
- #112: All 6 readers wrapped in try/except (corrupt files don't abort)
- #113: Collection cache invalidation after re-ingestion
- #114: Web UI message history capped at 200
- #115: Setup wizard config overwrite confirmation
- #116: Prompt injection defense (delimiter sanitization)
- #117: References section regex precision
- #118: Correction prompt truncation (1500 chars)
- #119: Initial generate() wrapped in try/except
- #120: Reasoning model detection (exact match + prefix)
- #121: make_fuzzy_query regex precision
- #122: REPLACE removed from dangerous keywords (function is read-only)
- #123: Non-dict YAML config rejection
- #124: Web UI model cache invalidation on provider change
- #125: Web UI uses fast file-based KB brief (was slow LLM call)
- #126: Re-ingestion progress message
- #127: Empty API key warning improved

Cross-checked by regression checker + design guardian agents.
Design score: 8.5 → 9.1/10. Tests: 174/174 passing.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

app_cli.py

@@ -3,6 +3,7 @@
 import sys
 
 from rich.console import Console
+from rich.markup import escape as rich_escape
 from rich.panel import Panel
 from rich.markdown import Markdown
 from rich.text import Text
@@ -11,7 +12,7 @@ from src.config_loader import load_config, get_api_key
 from src.ingest import ingest_documents
 from src.kb_meta import load_kb_meta_brief
 from src.query_engine import understand_query
-from src.retriever import retrieve
+from src.retriever import clear_collection_cache, retrieve
 from src.verifier import verify_and_respond
 from src.llm import list_models
 
@@ -256,9 +257,10 @@ def main() -> None:
             with console.status("Ingesting documents..."):
                 try:
                     count = ingest_documents(cfg)
+                    clear_collection_cache()
                     console.print(f"[green]Ingestion complete: {count} chunks.[/green]")
                 except Exception as e:
-                    console.print(f"[red]Ingestion error: {e}[/red]")
+                    console.print(f"[red]Ingestion error: {rich_escape(str(e))}[/red]")
             continue
 
         if result == "__MODEL__":
@@ -301,7 +303,7 @@ def main() -> None:
                         state.get("conversation_history", []),
                     )
                 except Exception as e:
-                    console.print(f"[yellow]Query understanding failed, using raw query: {e}[/yellow]")
+                    console.print(f"[yellow]Query understanding failed, using raw query: {rich_escape(str(e))}[/yellow]")
                     qu_result = {"action": "search", "search_query": user_input, "display_query": user_input, "original_query": user_input, "route": "vector", "sql_query": None}
 
             # Handle clarification
@@ -344,7 +346,7 @@ def main() -> None:
 
             # Show reformulated query if different from original
             if search_query != user_input:
-                console.print(f"[dim]Searching for: \"{search_query}\"[/dim]")
+                console.print(f"[dim]Searching for: \"{rich_escape(search_query)}\"[/dim]")
             if route in ("sql", "both"):
                 console.print(f"[dim]Using SQL query for structured data[/dim]")
 
@@ -353,7 +355,7 @@ def main() -> None:
             try:
                 retrieval_result = retrieve(search_query, effective_cfg, route=route, sql_query=sql_query)
             except Exception as e:
-                console.print(f"[red]Retrieval error: {e}[/red]")
+                console.print(f"[red]Retrieval error: {rich_escape(str(e))}[/red]")
                 continue
 
         state["last_retrieval"] = retrieval_result
@@ -366,13 +368,17 @@ def main() -> None:
                     original_query=user_input,
                 )
             except Exception as e:
-                console.print(f"[red]Generation error: {e}[/red]")
+                console.print(f"[red]Generation error: {rich_escape(str(e))}[/red]")
                 continue
 
         # ── Update conversation history ─────────────────────────────────
         history = state.get("conversation_history", [])
         max_history = qu_cfg.get("max_history", 6)
-        history.append({"role": "user", "content": user_input})
+        # Use display_query (post-QU) as the user message for history,
+        # since it captures clarification context and avoids duplicating
+        # the raw user_input that clarification already added.
+        effective_user_msg = display_query if display_query != user_input else user_input
+        history.append({"role": "user", "content": effective_user_msg})
         history.append({"role": "assistant", "content": result.get("response", "")})
         state["conversation_history"] = history[-max_history:]
 

app_web.py

@@ -4,9 +4,9 @@ import streamlit as st
 
 from src.config_loader import load_config, get_api_key
 from src.ingest import get_chroma_collection, ingest_documents
-from src.kb_meta import summarize_kb_for_welcome
+from src.kb_meta import load_kb_meta_brief
 from src.query_engine import understand_query
-from src.retriever import retrieve
+from src.retriever import clear_collection_cache, retrieve
 from src.verifier import verify_and_respond
 from src.llm import list_models
 
@@ -59,6 +59,13 @@ def render_sidebar():
             cfg.setdefault("llm", {})["provider"] = provider
 
         # --- Model dropdown (cached per provider) ---
+        # Invalidate model cache if provider changed
+        prev_provider_key = "prev_provider"
+        if st.session_state.get(prev_provider_key) != provider:
+            for p in providers:
+                st.session_state.pop(f"models_{p}", None)
+            st.session_state[prev_provider_key] = provider
+
         models_cache_key = f"models_{provider}"
         if models_cache_key not in st.session_state:
             api_key = get_api_key(cfg, provider)
@@ -115,9 +122,11 @@ def render_sidebar():
 
         # --- Re-ingest button ---
         if st.button("Re-ingest documents", use_container_width=True):
+            st.info("Ingestion may take a few minutes for large document collections...")
             with st.spinner("Ingesting documents..."):
                 try:
                     count = ingest_documents(cfg)
+                    clear_collection_cache()
                     st.success(f"Ingested {count} chunks.")
                     # Clear cached data so it refreshes after re-ingest
                     st.session_state.pop("kb_welcome_summary", None)
@@ -279,6 +288,11 @@ def render_chat():
             {"role": "assistant", "content": result.get("response", "")}
         )
 
+        # Cap message history to prevent unbounded memory growth
+        max_messages = 200  # 100 Q&A pairs
+        if len(st.session_state.messages) > max_messages:
+            st.session_state.messages = st.session_state.messages[-max_messages:]
+
 
 # ── Main ─────────────────────────────────────────────────────────────────────
 
@@ -300,7 +314,7 @@ def main():
     # Show KB summary on first visit (LLM-generated welcome summary)
     if not st.session_state.messages:
         if "kb_welcome_summary" not in st.session_state:
-            st.session_state.kb_welcome_summary = summarize_kb_for_welcome(cfg)
+            st.session_state.kb_welcome_summary = load_kb_meta_brief(cfg)
         kb_summary = st.session_state.kb_welcome_summary
         if kb_summary:
             with st.expander("Knowledge Base Contents", expanded=True):

docs/BUGFIX_LOG.md

@@ -186,7 +186,7 @@ severity, file, root cause, fix applied, and the audit round that found it.
 | K12 | LOW | Multiple | `cfg.get("section", {})` systemic pattern — if YAML section exists but is `None`, returns `None` not `{}` | Partially mitigated by config_loader normalization; remaining ~30 locations are low risk |
 | K6 | LOW | `src/search/semantic_scholar.py` | Filters out papers without abstracts, reducing result count for some topics | By design — abstract-less papers can't provide useful context |
 | K7 | LOW | `src/retriever.py` | LIKE wildcards `_`/`%` not escaped in fallback SQL query | Minor — extra matches unlikely to cause visible problems |
-| K8 | LOW | `CLAUDE.md` | Test count says 162, should be 165 | Doc update needed |
+| K8 | LOW | `CLAUDE.md` | Test count said 162, actual is 174 | FIXED (Round 11 doc update) |
 | K9 | LOW | `src/verifier.py` | `compute_similarity_flags` regex `[a-z]{3,}` excludes non-ASCII and 2-letter acronyms (AI, US, UK) | Advisory-only layer; impact limited to noisy flags |
 
 ---
@@ -240,7 +240,7 @@ This round was a comprehensive pre-release inspection using 6 parallel audit age
 
 | # | Severity | File | Description | Reason |
 |---|----------|------|-------------|--------|
-| K13 | HIGH | `src/llm/gemini.py` + `requirements.txt` | **CON-4**: `google-generativeai` package fully deprecated; replacement `google-genai` has different API surface | Requires full API migration; functional for now but will break when PyPI removes package |
+| K13 | ~~HIGH~~ | `src/llm/gemini.py` + `requirements.txt` | **CON-4**: `google-generativeai` package fully deprecated; replacement `google-genai` has different API surface | **FIXED** — Migrated to `google-genai>=1.0.0` SDK with `genai.Client(api_key=...)` (thread-safe, no global state) |
 | K14 | MEDIUM | `src/verifier.py` | **AH-05**: Token cap step function too generous for small contexts (501 chars → 1536 tokens = 12:1 ratio) | Would benefit from continuous formula; current 3-tier approach is functional |
 | K15 | MEDIUM | `src/verifier.py:24-33` | **AH-11**: Warning phrase list missing common patterns: "studies have shown", "research suggests", "historically" | Advisory layer only; expandable post-release |
 | K16 | LOW | `src/verifier.py:184` | **INSPECT-2**: References section regex matches "sources" in prose, not just section header | Mitigated: filename matching still works because match includes everything to end of response |
@@ -276,13 +276,212 @@ User-reported bug: querying "south korea in pts" returned zero SQL results despi
 
 ---
 
+## Round 11 — 2026-03-06 Session 6 (8-Agent Final Pre-Release Audit + Cross-Check)
+
+This round was the strictest final audit before public release: 8 parallel agents (core pipeline, ingestion/storage, SQL security, LLM/prompts, UIs/setup, readers/search, test suite, design docs/architecture) each read every line of their assigned files and cross-checked findings.
+
+### HIGH Fixes
+
+| # | Severity | File | Bug | Fix | Status |
+|---|----------|------|-----|-----|--------|
+| 84 | HIGH | `src/query_engine.py:96` | QU `max_tokens=256` too tight — SQL queries + reasoning fields truncate JSON output, causing silent fallback to vector-only search | Increased to `max_tokens=512` | FIXED |
+| 85 | HIGH | `src/llm/gemini.py:25-26` | Gemini `generate()` returned error strings instead of raising exceptions, inconsistent with OpenAI/Anthropic — error strings leaked into correction loop as "responses" | Changed to `raise RuntimeError(...)` for API errors (ValueError for safety blocks still returns string) | FIXED |
+| 86 | HIGH | `src/sql_retriever.py:184-214` | `make_fuzzy_query()` unsanitized value interpolation — captured regex values not escaped for single quotes, allowing SQL injection via crafted `WHERE` values | Added `.replace("'", "''")` escaping in both `_phrase_replace()` and `_word_replace()` | FIXED |
+| 87 | HIGH | `src/config_loader.py:35-37` | Nested null YAML values crashed consumers — `vector_db:` with no value → `.get("vector_db", "chroma_db")` returns `None` → `TypeError` in `os.path.isabs(None)` | Changed top-level-only normalization to recursive `_normalize_nulls()` that handles all nested dicts | FIXED |
+| 88 | HIGH | `setup.py:93-94` | `.env` parser used `strip('"')` which removes ALL leading/trailing quotes, not just one matched pair — could corrupt API keys starting/ending with quote chars | Changed to proper one-pair unwrap: check for matching pair then slice `[1:-1]` | FIXED |
+| 89 | HIGH | `setup.py:100-102` | `.env` writer did not escape special chars in values — API keys containing `"` or `\` produced malformed `.env` files | Added `v.replace('\\', '\\\\').replace('"', '\\"')` before writing | FIXED |
+| 90 | HIGH | `src/sql_retriever.py:41-42` | `sqlite_master` access not blocked — `SELECT * FROM sqlite_master` passed all validation, exposing full DDL schema | Added `sqlite_(master\|schema\|temp_master\|temp_schema)` check in `_validate_sql()` | FIXED |
+| 91 | HIGH | `tests/test_verifier.py:65-66,95-96` | Wrong mock target — patched `src.kb_meta.load_kb_meta` but verifier imports `load_kb_meta_brief`; mocks were no-ops (only worked due to fallback chain) | Changed both patches to `src.kb_meta.load_kb_meta_brief` | FIXED |
+| 92 | HIGH | `src/retriever.py:381` | SQL keyword fallback ran unconditionally for `route="sql"` even when vector search already found results — wasted computation | Added `and not db_results` condition: only run keyword SQL when vector fallback also found nothing | FIXED |
+
+### MEDIUM Fixes
+
+| # | Severity | File | Bug | Fix | Status |
+|---|----------|------|-----|-----|--------|
+| 93 | MEDIUM | `src/search/semantic_scholar.py:47,49` | `citationCount: null` from API → `None` in sort key → `TypeError: '<' not supported between NoneType and int` crash | Changed to `paper.get("citationCount") or 0` and `x.get("citation_count") or 0` | FIXED |
+| 94 | MEDIUM | `app_web.py:123` | Stale KB welcome summary after re-ingest — `st.session_state.kb_welcome_summary` not cleared before `st.rerun()` | Added `st.session_state.pop("kb_welcome_summary", None)` before rerun | FIXED |
+| 95 | MEDIUM | `app_web.py:256-257,267` | SQL result count missing from web UI status label — showed "0 local + 0 web" even when SQL returned rows | Added `n_sql` to both "Verified" and "Done" status labels, matching CLI behavior | FIXED |
+| 96 | MEDIUM | `src/verifier.py:184` | References regex `\b(references\|sources)\b` matched casual mentions of "sources" in prose — citation audit checked entire response body instead of just References section | Changed to heading-specific regex: `^#+\s*(references\|sources)` or `^\*\*(references\|sources)\*\*` | FIXED |
+| 97 | MEDIUM | `src/verifier.py:486` | Non-strict mode used stale `citation_warnings` from last loop iteration, not from final corrected response | Added `validate_citations(response, retrieval_result)` recomputation before building warning | FIXED |
+| 98 | MEDIUM | `src/ingest.py:291-298` | Meta files deleted before overview regeneration — if `build_and_store_overview()` failed, KB lost self-awareness until next ingestion | Removed pre-deletion; `write_text()` in `build_and_store_overview()` already overwrites | FIXED |
+| 99 | MEDIUM | `src/readers/docx.py:4-10` | DOCX reader only extracted paragraphs, silently dropping all embedded tables | Added table extraction loop: `doc.tables` → row cells joined with ` \| ` | FIXED |
+
+### Known Issues Added
+
+| # | Severity | File | Description | Reason |
+|---|----------|------|-------------|--------|
+| K18 | MEDIUM | `src/sql_retriever.py:17-24` | Comment stripping is naive (regex) — corrupts string literals containing `--` or `/* */` | Rare in LLM-generated queries; proper state-machine parser would be over-engineering |
+| K19 | MEDIUM | `src/retriever.py:195-196, 267-269` | LIKE wildcard chars `%` and `_` not escaped in fallback/alt-column queries | Same as existing K7; low real-world impact |
+| K20 | MEDIUM | `src/sql_retriever.py` | No SQL execution timeout — Cartesian products could hang | Same as existing K10; SQLite lacks built-in SELECT timeout |
+| K21 | MEDIUM | `src/verifier.py:151-202` | `validate_citations` compares citation numbers against raw retrieval count, not References section entry count | Would require parsing References section to count entries; current check is conservative |
+| K22 | LOW | `src/readers/csv_tab.py:11` | CSV encoding hardcoded to `utf-8-sig` — non-UTF-8 files get replacement chars | Would need `chardet` dependency or encoding detection |
+| K23 | LOW | `src/sql_ingest.py:568-586` | Row-by-row SQL INSERT slow for large datasets | Would benefit from `executemany()` batching |
+| K24 | LOW | `src/readers/excel.py:40, csv_tab.py:22` | `zip(headers, row)` silently truncates extra columns in ragged rows | Acceptable for most research datasets |
+| K25 | LOW | `src/ingest.py:104` | `rglob("*")` follows symlinks — cyclic symlinks could cause infinite loop | Rare in knowledge_base directories |
+| K26 | LOW | Test suite | Zero coverage: `validate_citations`, verification loop, `make_fuzzy_query`, `compute_similarity_flags`, `load_kb_meta_brief` | Test gap documented; non-blocking for release |
+
+### Test Coverage Gaps (Confirmed)
+
+| Priority | What | Description |
+|----------|------|-------------|
+| HIGH | `validate_citations()` | Layer 4.5 — zero direct tests |
+| HIGH | Verification correction loop | Core feature — zero multi-iteration tests |
+| HIGH | `make_fuzzy_query()` | SQL fuzzy fallback — zero tests |
+| MEDIUM | `compute_similarity_flags()` | Layer 4 — zero tests |
+| MEDIUM | `load_kb_meta_brief()` | Used by verifier — zero tests |
+
+---
+
+## Round 12 — 2026-03-06 Session 7 (Pre-Release Final Audit)
+
+See `docs/AUDIT_BUGLOG.md` Round 12 section for full details.
+33 bugs found across 3 tiers, all FIXED. Design score raised from 7.1/10 to 9/10.
+5 NEW low-severity issues documented as KNOWN. 174/174 tests passing. **Released to GitHub.**
+
+---
+
+## Round 13 — 2026-03-07 (Post-Release Deep Audit)
+
+**7 parallel audit agents** (design evaluation + core pipeline + verification stack + SQL layer + LLM/readers/search + application layer + test coverage). Full line-by-line review of all 25 source files, 17 test files, and all documentation.
+
+**Design Score**: 8.5/10 → **9.1/10** (post-fix) | **Test Coverage Score**: 6/10
+
+### Tier 1: Anti-Hallucination Integrity (4 bugs) — ALL FIXED
+
+| # | Severity | File | Bug | Fix | Status |
+|---|----------|------|-----|-----|--------|
+| 100 | HIGH | `verifier.py` | **R13-T1-01**: Verification LLM persistent failure → `except Exception: continue` skips all iterations → non-strict returns unverified response. Full bypass of Layers 3–5. | Added `any_verification_ran` flag. If ALL verification LLM calls fail: strict mode refuses with system-failure message; non-strict adds prominent WARNING banner distinguishing "could not run" from "did not pass". | FIXED |
+| 101 | HIGH | `verifier.py` | **R13-T1-02**: `validate_citations` SQL/web source matching uses wrong dict structure (dead code). Layer 4.5 only works for local sources. | SQL: extract source filename from context via regex. Web: changed to flat dict access `web_chunk.get("url", "")`. | FIXED |
+| 102 | HIGH | `verifier.py` | **R13-T1-03**: Citation count uses unique source *files*, not citable *units*. False "fabricated reference" warning on multi-chunk single-file sources. | Changed `db_count = len(db_results)` (chunks) instead of `len(unique_db_sources)` (files). Each chunk is a citable unit. | FIXED |
+| 103 | MEDIUM | `verifier.py` | **R13-T1-04**: `max_iterations: 0` with `enabled: true` silently disables verification. | Clamp `max_iterations = max(1, max_iterations)` with warning. Must use `enabled: false` to disable. Test updated. | FIXED |
+
+### Tier 2: Pre-Release Quality (24 bugs) — ALL FIXED
+
+| # | Severity | File | Bug | Fix | Status |
+|---|----------|------|-----|-----|--------|
+| 104 | HIGH | `retriever.py` | **R13-T2-01**: Missing LIKE wildcard ESCAPE in `_build_fallback_sql_query` and `_try_alternate_columns`. | Added `%`/`_` escaping and `ESCAPE '\'` clause. Initial fix had double-backslash bug; corrected by regression checker. | FIXED |
+| 105 | HIGH | `gemini.py` | **R13-T2-02**: Gemini safety filter returns `"[Gemini blocked: ...]"` bypassing anti-hallucination stack. | Return `""` instead. Verifier's empty-response handler (lines 369-378) provides user-facing refusal. | FIXED |
+| 106 | HIGH | `openai.py` | **R13-T2-03**: Reasoning model budget uncapped → API error with large `max_tokens`. | `reasoning_budget = min(max(max_tokens * 4, 4096), 128000)`. | FIXED |
+| 107 | HIGH | `llm/__init__.py` | **R13-T2-04**: Non-numeric temperature/max_tokens crashes with opaque ValueError. | Wrapped in try/except with defaults (0.0 and 2048). | FIXED |
+| 108 | HIGH | `app_cli.py` | **R13-T2-05**: After clarification, raw `user_input` re-appended → duplicate history entries. | Use `display_query` (post-QU, includes clarification context) instead of raw `user_input`. | FIXED |
+| 109 | MEDIUM | `llm/__init__.py` | **R13-T2-06**: Anthropic temp > 1.0 → API error. | Added provider-specific clamp: `if provider == "anthropic": temperature = min(temperature, 1.0)`. | FIXED |
+| 110 | MEDIUM | `sql_retriever.py` | **R13-T2-07**: No SQL query timeout. `CROSS JOIN` hangs indefinitely. | Added `set_progress_handler` with 10-second abort. `import time` at module level. | FIXED |
+| 111 | MEDIUM | `app_cli.py` | **R13-T2-08**: Rich markup injection via brackets in error messages/queries. | Added `rich_escape()` to all 5 interpolated f-strings in the RAG pipeline loop. | FIXED |
+| 112 | MEDIUM | `readers/*.py` | **R13-T2-09**: No try/except in readers. Corrupt file aborts ingestion. | Wrapped `read_pdf`, `read_docx`, `read_excel`, `read_stata`, `read_rdata`, `read_spss` in try/except. | FIXED |
+| 113 | MEDIUM | `retriever.py`, `app_cli.py`, `app_web.py` | **R13-T2-10**: `_collection_cache` never invalidated after re-ingestion. | Added `clear_collection_cache()` function; called after ingestion in both CLI and web. | FIXED |
+| 114 | MEDIUM | `app_web.py` | **R13-T2-11**: Web UI messages grow without limit. | Capped at 200 entries (100 Q&A pairs) after each append cycle. | FIXED |
+| 115 | MEDIUM | `setup.py` | **R13-T2-12**: Setup wizard overwrites config without confirmation. | Added existence check + `[y/N]` prompt before overwriting `config.yaml`. | FIXED |
+| 116 | MEDIUM | `prompts.py` | **R13-T2-13**: Prompt injection via KB delimiter mimicry. | Sanitize context: replace 69-char `=` sequences with Unicode `≡` before injection. | FIXED |
+| 117 | MEDIUM | `verifier.py` | **R13-T2-14**: References regex matches `## Data Sources` etc. | Anchored regex with `\s*$` to require standalone headers only. | FIXED |
+| 118 | MEDIUM | `verifier.py` | **R13-T2-15**: Correction prompt includes full previous response. | Truncate to 1500 chars. Instruction: "Rewrite COMPLETE response from scratch". | FIXED |
+| 119 | MEDIUM | `verifier.py` | **R13-T2-16**: Initial `generate()` exception propagates. | Wrapped in try/except; returns `refused: True` with error message. | FIXED |
+| 120 | MEDIUM | `openai.py` | **R13-T2-17**: Reasoning model detection via brittle prefix match. | Changed to `model in ("o1","o3","o4") or model.startswith(("o1-","o3-","o4-"))`. | FIXED |
+| 121 | MEDIUM | `sql_retriever.py` | **R13-T2-18**: `make_fuzzy_query` regex matches non-column contexts. | Pattern now requires quoted identifiers or valid SQL column names. | FIXED |
+| 122 | MEDIUM | `sql_retriever.py` | **R13-T2-19**: `REPLACE()` function falsely blocked. | Removed `REPLACE` from `_DANGEROUS_KEYWORDS`. Statement form blocked by SELECT-only check. | FIXED |
+| 123 | MEDIUM | `config_loader.py` | **R13-T2-20**: Non-dict YAML → `AttributeError`. | Added `isinstance(cfg, dict)` check with clear ValueError message. | FIXED |
+| 124 | MEDIUM | `app_web.py` | **R13-T2-21**: Model list cache stale after provider change. | Added provider-change detection that clears all model caches. | FIXED |
+| 125 | MEDIUM | `app_web.py` | **R13-T2-22**: Web UI uses slow LLM call for KB welcome. | Changed to `load_kb_meta_brief()` (file read) matching CLI behavior. | FIXED |
+| 126 | MEDIUM | `app_web.py` | **R13-T2-23**: Re-ingestion blocks without progress feedback. | Added `st.info()` message before spinner to set expectations. | FIXED |
+| 127 | MEDIUM | `setup.py` | **R13-T2-24**: Empty API key warning not actionable. | Improved message: "Set it later by editing .env or re-running: python setup.py". | FIXED |
+
+### Tier 3: Post-Release Improvements (20 bugs)
+
+| # | Severity | File | Bug | Status |
+|---|----------|------|-----|--------|
+| 128 | MED | `openai.py:9`, `anthropic.py:9`, `gemini.py:16` | **R13-T3-01**: New API client per `generate()` call. No connection reuse. | OPEN |
+| 129 | MED | `llm/openai.py:52`, `anthropic.py:32`, `gemini.py:48` | **R13-T3-02**: `list_models()` returns hardcoded list, not live API fetch. Design doc says otherwise. | OPEN |
+| 130 | MED | `retriever.py:282-285` | **R13-T3-03**: `_try_alternate_columns` quote escaping logic fragile and non-obvious. | OPEN |
+| 131 | MED | `sql_retriever.py:54` | **R13-T3-04**: `sqlite_master` check on `stripped` instead of `unquoted`. False positives. | OPEN |
+| 132 | MED | `readers/stata.py:28`, `rdata.py:15` | **R13-T3-05**: `df.iterrows()` 10-100x slower than `itertuples()`. Large datasets take minutes. | OPEN |
+| 133 | LOW | `sql_retriever.py:26-33` | **R13-T3-06**: `_strip_sql_comments` removes `--` inside string literals. Confirmed from R12 NEW-04. | OPEN |
+| 134 | LOW | `verifier.py:429` | **R13-T3-07**: Verification LLM system prompt minimal. No anti-hallucination rules for verifier itself. | OPEN |
+| 135 | LOW | `verifier.py:25-34` | **R13-T3-08**: Warning phrases easily evaded by paraphrasing. Only 8 exact-match phrases. | OPEN |
+| 136 | LOW | `verifier.py:126` | **R13-T3-09**: Term overlap excludes 2-char acronyms (AI, UK, EU, US, UN). | OPEN |
+| 137 | LOW | `verifier.py:181` | **R13-T3-10**: Citation `[0]` never detected as invalid. | OPEN |
+| 138 | LOW | `readers/csv_tab.py:11`, `text.py:5` | **R13-T3-11**: Hardcoded UTF-8. Non-UTF-8 files get replacement chars. | OPEN |
+| 139 | LOW | `readers/docx.py:24` | **R13-T3-12**: All DOCX content mapped to `page: 1`. | OPEN |
+| 140 | LOW | `readers/pdf.py:9` | **R13-T3-13**: Scanned PDFs silently return empty list. No warning. | OPEN |
+| 141 | LOW | `readers/excel.py:15` | **R13-T3-14**: Unevaluated formulas appear as `None`, silently dropped. | OPEN |
+| 142 | LOW | `search/semantic_scholar.py:15-28` | **R13-T3-15**: Retries on non-retriable 4xx errors. Wastes 17s. | OPEN |
+| 143 | LOW | `search/__init__.py:10-12` | **R13-T3-16**: Unknown backend names silently return `[]`. | OPEN |
+| 144 | LOW | `setup.py:96-100` | **R13-T3-17**: `.env` unescape wrong order. Theoretical corruption. | OPEN |
+| 145 | LOW | `config_loader.py:20-22` | **R13-T3-18**: `.env` loaded before config existence check. | OPEN |
+| 146 | LOW | `app_web.py:288` | **R13-T3-19**: Browser tab title hardcoded "ResearchBot". | OPEN |
+| 147 | LOW | `verifier.py:460-462` | **R13-T3-20**: Correction call uses same temperature as initial. Should force 0.0. | OPEN |
+
+### Test Coverage Gaps (Round 13 — confirmed + expanded)
+
+| Priority | What | File | Status |
+|----------|------|------|--------|
+| CRITICAL | `validate_citations()` — Layer 4.5 | `verifier.py:152-230` | OPEN |
+| CRITICAL | Verification correction loop | `verifier.py:407-520` | OPEN |
+| CRITICAL | `make_fuzzy_query()` | `sql_retriever.py:181-237` | OPEN |
+| HIGH | `compute_similarity_flags()` — Layer 4 | `verifier.py:101-149` | OPEN |
+| HIGH | `_strip_quoted()` / `_strip_sql_comments()` | `sql_retriever.py:17-33` | OPEN |
+| HIGH | Empty LLM response in `verify_and_respond` | `verifier.py:354-364` | OPEN |
+| HIGH | `build_and_store_overview()` orchestrator | `kb_meta.py:464-492` | OPEN |
+| MEDIUM | `generate_brief_overview()` | `kb_meta.py:385-422` | OPEN |
+| MEDIUM | `load_kb_meta_brief()` fallback | `kb_meta.py:434-445` | OPEN |
+| MEDIUM | `summarize_kb_for_welcome()` | `kb_meta.py:514-552` | OPEN |
+| MEDIUM | `_build_fallback_sql_query` with real data | `retriever.py:159-217` | OPEN |
+| MEDIUM | `_try_alternate_columns` with real data | `retriever.py:220-292` | OPEN |
+| MEDIUM | `_validate_sql` edge cases | `sql_retriever.py:36-59` | OPEN |
+| MEDIUM | `format_sql_results_as_context` truncation | `sql_retriever.py:116-167` | OPEN |
+| MEDIUM | `_normalize_nulls` | `config_loader.py:36-42` | OPEN |
+
+### Cross-Agent Consensus (confirmed by 2+ agents independently)
+
+| Bug | Agents | Confidence |
+|-----|--------|-----------|
+| `validate_citations` dead code for SQL + web (#101) | Core Pipeline, Application, Verification | **Very High** |
+| Missing LIKE ESCAPE in SQL fallbacks (#104) | SQL, Core Pipeline, Design | **Very High** |
+| Stale `_collection_cache` after re-ingestion (#113) | Core Pipeline, Design | **High** |
+| Verification LLM failure → silent bypass (#100) | Verification, Core Pipeline | **High** |
+| Missing reader exception handling (#112) | LLM/Readers, Design | **High** |
+
+### Recommended Fix Order
+
+**Phase 1 — Anti-hallucination integrity (#100–#103):**
+Fix verification bypass, validate_citations data structure, citation counting, max_iterations guard.
+
+**Phase 2 — High-severity T2 bugs (#104–#108):**
+LIKE ESCAPE, Gemini blocked response, reasoning model overflow, config validation, history corruption.
+
+**Phase 3 — Test coverage (3 CRITICAL gaps):**
+Tests for validate_citations, verification correction loop, make_fuzzy_query.
+
+**Phase 4 — Remaining T2 MEDIUM bugs (#109–#127):**
+Address in order of impact.
+
+**Phase 5 — T3 improvements (#128–#147):**
+Address as time permits.
+
+---
+
+## Round 14 — 2026-03-07 (Tier 1 + Tier 2 Bug Fix Sprint)
+
+**6 parallel agents**: 4 fix agents (T1, T2-HIGH, T2-MED-A, T2-MED-B) + 1 regression checker + 1 design guardian.
+
+- Fixed all 28 Tier 1 + Tier 2 bugs (#100–#127)
+- Regression checker found 1 CRITICAL issue in Bug #104 fix (double-backslash in ESCAPE clause) — corrected
+- Also fixed `spss.py` missing try/except (pre-existing gap found by regression checker)
+- Design Guardian assessment: design score 8.5 → 9.1/10, recommendation: SHIP
+- Updated test for Bug #103 (renamed to `test_max_iterations_zero_clamps_to_one`)
+- 174/174 tests passing
+
+---
+
 ## Statistics
 
 | Metric | Count |
 |--------|-------|
-| Total bugs fixed | ~119 (3 this round) |
-| Audit rounds completed | 10 (10 sessions, 30+ total agent dispatches) |
+| Total bugs found (all rounds) | ~216 (48 new in Round 13) |
+| Total bugs fixed (all rounds) | ~196 (168 in R1–R12, 28 in R14) |
+| Round 13 T1+T2 bugs | 28 FIXED (was 28 OPEN) |
+| Round 13 T3 bugs remaining | 20 OPEN (5 MEDIUM, 15 LOW) |
+| Audit rounds completed | 14 (50+ total agent dispatches) |
 | Test count | 174 passing |
 | Test files | 17 |
-| Source files audited | All 20+ |
-| Known issues remaining | 17 (0 CRITICAL, 1 HIGH, 6 MEDIUM, 10 LOW) |
+| Source files modified in R14 | 16 (verifier, retriever, gemini, openai, llm/__init__, app_cli, app_web, sql_retriever, config_loader, prompts, setup, pdf, docx, excel, stata, rdata, spss) |
+| Design score | 9.1/10 (up from 8.5) |
+| CRITICAL test gaps | 3 (validate_citations, correction loop, make_fuzzy_query) |

setup.py

@@ -141,7 +141,8 @@ def run_wizard():
     # Step 3b: API key
     api_key = getpass.getpass(f"Enter your {provider} API key: ")
     if not api_key:
-        print(f"  Warning: No API key entered for {provider}. You can set it later in .env")
+        print(f"  Warning: No API key entered for {provider}.")
+        print(f"  Set it later by editing .env or re-running: python setup.py")
 
     # Step 4: Model selection — fetch available models
     print(f"\nFetching available {provider} models...")
@@ -179,9 +180,18 @@ def run_wizard():
 
     # Write config.yaml
     config_path = project_root / "config.yaml"
-    config_str = generate_config(bot_name, domain, provider, model, web_search)
-    config_path.write_text(config_str)
-    print(f"\n  Wrote {config_path}")
+    if config_path.exists():
+        overwrite = input(f"\n{config_path} already exists. Overwrite? [y/N]: ").strip().lower()
+        if overwrite != 'y':
+            print("  Keeping existing config.yaml.")
+        else:
+            config_str = generate_config(bot_name, domain, provider, model, web_search)
+            config_path.write_text(config_str)
+            print(f"\n  Wrote {config_path}")
+    else:
+        config_str = generate_config(bot_name, domain, provider, model, web_search)
+        config_path.write_text(config_str)
+        print(f"\n  Wrote {config_path}")
 
     # Write .env (merges with existing keys if present)
     env_path = project_root / ".env"

src/config_loader.py

@@ -30,6 +30,12 @@ def load_config(config_path: str = None) -> dict:
     with open(config_path, "r", encoding="utf-8") as f:
         cfg = yaml.safe_load(f) or {}
 
+    if not isinstance(cfg, dict):
+        raise ValueError(
+            f"Config file must contain a YAML mapping (dict), got {type(cfg).__name__}. "
+            "Run 'python setup.py' to regenerate config.yaml."
+        )
+
     # Normalize None-valued sections to empty dicts so chained .get() never
     # fails with AttributeError (e.g. `llm:` with no sub-keys → None).
     # Recurse into nested dicts so `paths:\n  vector_db:` also gets normalized.

src/llm/__init__.py

@@ -24,8 +24,17 @@ def generate(system_prompt: str, user_message: str, cfg: dict,
     if max_tokens is None:
         max_tokens = llm_cfg.get("max_tokens", 2048)
     # Validate parameters
-    temperature = max(0.0, min(float(temperature), 2.0))
-    max_tokens = max(1, min(int(max_tokens), 128000))
+    try:
+        temperature = max(0.0, min(float(temperature), 2.0))
+    except (ValueError, TypeError):
+        temperature = 0.0
+    # Anthropic max temperature is 1.0
+    if provider == "anthropic":
+        temperature = min(temperature, 1.0)
+    try:
+        max_tokens = max(1, min(int(max_tokens), 128000))
+    except (ValueError, TypeError):
+        max_tokens = 2048
     return PROVIDERS[provider].generate(
         system_prompt=system_prompt, user_message=user_message,
         api_key=api_key, model=model, temperature=temperature, max_tokens=max_tokens,

src/llm/gemini.py

@@ -30,8 +30,8 @@ def generate(system_prompt: str, user_message: str, api_key: str,
             return ""
         return response.text
     except ValueError as e:
-        # Safety filter block
-        return f"[Gemini blocked: {e}]"
+        # Safety filter block — return empty so verifier treats it as generation failure
+        return ""
     except Exception as e:
         error_type = type(e).__name__
         error_msg = str(e)

src/llm/openai.py

@@ -7,12 +7,12 @@ def generate(system_prompt: str, user_message: str, api_key: str,
     model = model or "gpt-4o"
     from openai import OpenAI
     client = OpenAI(api_key=api_key)
-    is_reasoning = model.startswith(("o1", "o3", "o4"))
+    is_reasoning = model in ("o1", "o3", "o4") or model.startswith(("o1-", "o3-", "o4-"))
     try:
         if is_reasoning:
             # Reasoning models include thinking tokens in max_completion_tokens,
             # so we need a larger budget to get enough output tokens.
-            reasoning_budget = max(max_tokens * 4, 4096)
+            reasoning_budget = min(max(max_tokens * 4, 4096), 128000)
             messages = [
                 {"role": "developer", "content": system_prompt},
                 {"role": "user", "content": user_message},

src/prompts.py

@@ -299,10 +299,13 @@ def build_prompt(
     else:
         overview_section = ""
 
+    # Sanitize context to prevent prompt injection via delimiter mimicry
+    safe_context = context.replace("=" * 69, "\u2261" * 69)
+
     return SYSTEM_PROMPT_TEMPLATE.format(
         bot_name=bot_name,
         domain=domain,
-        context=context,
+        context=safe_context,
         kb_overview_section=overview_section,
     )
 

src/readers/docx.py

@@ -2,23 +2,27 @@
 
 
 def read_docx(file_path: str) -> list[dict]:
-    from docx import Document
-    doc = Document(file_path)
-    paragraphs = [p.text.strip() for p in doc.paragraphs if p.text.strip()]
+    try:
+        from docx import Document
+        doc = Document(file_path)
+        paragraphs = [p.text.strip() for p in doc.paragraphs if p.text.strip()]
 
-    # Extract embedded tables
-    for table in doc.tables:
-        for row in table.rows:
-            seen_elements = set()
-            cells = []
-            for cell in row.cells:
-                if id(cell._element) not in seen_elements:
-                    seen_elements.add(id(cell._element))
-                    if cell.text.strip():
-                        cells.append(cell.text.strip())
-            if cells:
-                paragraphs.append(" | ".join(cells))
+        # Extract embedded tables
+        for table in doc.tables:
+            for row in table.rows:
+                seen_elements = set()
+                cells = []
+                for cell in row.cells:
+                    if id(cell._element) not in seen_elements:
+                        seen_elements.add(id(cell._element))
+                        if cell.text.strip():
+                            cells.append(cell.text.strip())
+                if cells:
+                    paragraphs.append(" | ".join(cells))
 
-    if not paragraphs:
+        if not paragraphs:
+            return []
+        return [{"page": 1, "text": "\n\n".join(paragraphs)}]
+    except Exception as e:
+        print(f"Warning: Could not read {file_path}: {e}")
         return []
-    return [{"page": 1, "text": "\n\n".join(paragraphs)}]

src/readers/excel.py

@@ -2,11 +2,15 @@
 MAX_CHUNK_CHARS = 6000
 
 def read_excel(file_path: str) -> list[dict]:
-    from pathlib import Path
-    ext = Path(file_path).suffix.lower()
-    if ext == ".xls":
-        return _read_xls(file_path)
-    return _read_xlsx(file_path)
+    try:
+        from pathlib import Path
+        ext = Path(file_path).suffix.lower()
+        if ext == ".xls":
+            return _read_xls(file_path)
+        return _read_xlsx(file_path)
+    except Exception as e:
+        print(f"Warning: Could not read {file_path}: {e}")
+        return []
 
 def _read_xlsx(file_path: str) -> list[dict]:
     import openpyxl

src/readers/pdf.py

@@ -2,11 +2,15 @@
 
 
 def read_pdf(file_path: str) -> list[dict]:
-    from pypdf import PdfReader
-    reader = PdfReader(file_path)
-    pages = []
-    for i, page in enumerate(reader.pages):
-        text = page.extract_text()
-        if text and text.strip():
-            pages.append({"page": i + 1, "text": text.strip()})
-    return pages
+    try:
+        from pypdf import PdfReader
+        reader = PdfReader(file_path)
+        pages = []
+        for i, page in enumerate(reader.pages):
+            text = page.extract_text()
+            if text and text.strip():
+                pages.append({"page": i + 1, "text": text.strip()})
+        return pages
+    except Exception as e:
+        print(f"Warning: Could not read {file_path}: {e}")
+        return []

src/readers/rdata.py

@@ -2,34 +2,38 @@
 MAX_CHUNK_CHARS = 6000
 
 def read_rdata(file_path: str) -> list[dict]:
-    import pyreadr
-    result = pyreadr.read_r(file_path)
-    pages = []
-    for name, df in result.items():
-        name_display = name if name is not None else "data"
-        headers = list(df.columns)
-        header_line = f"Object: {name_display} | Columns: {', '.join(headers)}\n"
-        block = []
-        block_chars = len(header_line)
-        block_start = 1
-        for idx, (_, row) in enumerate(df.iterrows()):
-            parts = []
-            for col in headers:
-                val = row[col]
-                if val is not None and str(val).strip() and str(val).lower() not in ("nan", "nat", "<na>", "inf", "-inf"):
-                    parts.append(f"{col}: {val}")
-            if not parts:
-                continue
-            row_text = "; ".join(parts)
-            if block and block_chars + len(row_text) + 1 > MAX_CHUNK_CHARS:
+    try:
+        import pyreadr
+        result = pyreadr.read_r(file_path)
+        pages = []
+        for name, df in result.items():
+            name_display = name if name is not None else "data"
+            headers = list(df.columns)
+            header_line = f"Object: {name_display} | Columns: {', '.join(headers)}\n"
+            block = []
+            block_chars = len(header_line)
+            block_start = 1
+            for idx, (_, row) in enumerate(df.iterrows()):
+                parts = []
+                for col in headers:
+                    val = row[col]
+                    if val is not None and str(val).strip() and str(val).lower() not in ("nan", "nat", "<na>", "inf", "-inf"):
+                        parts.append(f"{col}: {val}")
+                if not parts:
+                    continue
+                row_text = "; ".join(parts)
+                if block and block_chars + len(row_text) + 1 > MAX_CHUNK_CHARS:
+                    text = header_line + "\n".join(block)
+                    pages.append({"page": f"{name_display}_rows_{block_start}-{block_start + len(block) - 1}", "text": text})
+                    block = []
+                    block_chars = len(header_line)
+                    block_start = idx + 1
+                block.append(row_text)
+                block_chars += len(row_text) + 1
+            if block:
                 text = header_line + "\n".join(block)
                 pages.append({"page": f"{name_display}_rows_{block_start}-{block_start + len(block) - 1}", "text": text})
-                block = []
-                block_chars = len(header_line)
-                block_start = idx + 1
-            block.append(row_text)
-            block_chars += len(row_text) + 1
-        if block:
-            text = header_line + "\n".join(block)
-            pages.append({"page": f"{name_display}_rows_{block_start}-{block_start + len(block) - 1}", "text": text})
-    return pages
+        return pages
+    except Exception as e:
+        print(f"Warning: Could not read {file_path}: {e}")
+        return []

src/readers/spss.py

@@ -2,6 +2,10 @@
 from src.readers.stata import _dataframe_to_pages
 
 def read_spss(file_path: str) -> list[dict]:
-    import pyreadstat
-    df, meta = pyreadstat.read_sav(file_path)
-    return _dataframe_to_pages(df, meta)
+    try:
+        import pyreadstat
+        df, meta = pyreadstat.read_sav(file_path)
+        return _dataframe_to_pages(df, meta)
+    except Exception as e:
+        print(f"Warning: Could not read {file_path}: {e}")
+        return []

src/readers/stata.py

@@ -2,9 +2,13 @@
 MAX_CHUNK_CHARS = 6000
 
 def read_stata(file_path: str) -> list[dict]:
-    import pyreadstat
-    df, meta = pyreadstat.read_dta(file_path)
-    return _dataframe_to_pages(df, meta)
+    try:
+        import pyreadstat
+        df, meta = pyreadstat.read_dta(file_path)
+        return _dataframe_to_pages(df, meta)
+    except Exception as e:
+        print(f"Warning: Could not read {file_path}: {e}")
+        return []
 
 def _dataframe_to_pages(df, meta) -> list[dict]:
     pages = []

src/retriever.py

@@ -7,6 +7,11 @@ from src.search import search, format_web_results_as_context
 _collection_cache = {}
 
 
+def clear_collection_cache():
+    """Clear the cached ChromaDB collection so it's re-read after re-ingestion."""
+    _collection_cache.clear()
+
+
 def _get_cached_collection(cfg: dict):
     """Get or create cached ChromaDB collection."""
     from src.ingest import get_chroma_collection
@@ -208,7 +213,8 @@ def _build_fallback_sql_query(query: str, cfg: dict) -> str | None:
                 safe_word = word.replace("'", "''")
                 if not re.match(r'^\w+$', safe_word):
                     continue  # Skip non-alphanumeric words
-                conditions.append(f'"{col}" LIKE \'%{safe_word}%\'')
+                safe_word = safe_word.replace('%', '\\%').replace('_', '\\_')
+                conditions.append(f'"{col}" LIKE \'%{safe_word}%\' ESCAPE \'\\\'')
 
         if conditions:
             where_clause = " OR ".join(conditions)
@@ -283,8 +289,9 @@ def _try_alternate_columns(sql_query: str, cfg: dict) -> tuple[list[dict], str]
     # Strip dangerous characters for defense-in-depth
     if not re.match(r"^[\w\s.,'-]+$", safe_value):
         safe_value = re.sub(r"[;'\\\"]", "", safe_value)
+    safe_value = safe_value.replace('%', '\\%').replace('_', '\\_')
     for col in text_cols:
-        alt_query = f'SELECT * FROM "{table_name}" WHERE "{col}" LIKE \'%{safe_value}%\' LIMIT {max_rows}'
+        alt_query = f'SELECT * FROM "{table_name}" WHERE "{col}" LIKE \'%{safe_value}%\' ESCAPE \'\\\' LIMIT {max_rows}'
         rows = execute_sql_query(alt_query, cfg)
         if rows:
             return rows, alt_query

src/sql_retriever.py

@@ -3,12 +3,13 @@
 import os
 import re
 import sqlite3
+import time
 from pathlib import Path
 
 
 _DANGEROUS_KEYWORDS = re.compile(
     r"\b(INSERT|UPDATE|DELETE|DROP|ALTER|CREATE|ATTACH|DETACH|PRAGMA|"
-    r"LOAD_EXTENSION|UNION|REPLACE|VACUUM|REINDEX|SAVEPOINT|RELEASE|"
+    r"LOAD_EXTENSION|UNION|VACUUM|REINDEX|SAVEPOINT|RELEASE|"
     r"ROLLBACK|BEGIN|COMMIT|GRANT|REVOKE|EXPLAIN|WITH)\b",
     re.IGNORECASE,
 )
@@ -101,9 +102,16 @@ def execute_sql_query(sql_query: str, cfg: dict) -> list[dict]:
     max_rows = cfg.get("sql", {}).get("max_rows", 200)
 
     try:
-        conn = sqlite3.connect(f"file:{db_path}?mode=ro", uri=True)
+        conn = sqlite3.connect(f"file:{db_path}?mode=ro", uri=True, timeout=5)
         try:
             conn.row_factory = sqlite3.Row
+            # Abort long-running queries after 10 seconds
+            _start = time.monotonic()
+            def _progress_check():
+                if time.monotonic() - _start > 10:
+                    return 1  # non-zero = abort
+                return 0
+            conn.set_progress_handler(_progress_check, 10000)
             cursor = conn.execute(sql_query)
             rows = cursor.fetchmany(max_rows)
             return [dict(row) for row in rows]
@@ -195,7 +203,7 @@ def make_fuzzy_query(sql: str, word_level: bool = False) -> str | None:
     Returns the modified query, or None if no substitutions were made.
     """
     pattern = re.compile(
-        r"""(["']?\w+["']?\s*)=\s*'([^']+)'""",
+        r"""("[^"]+"\s*|[A-Za-z_]\w*\s*)=\s*'([^']+)'""",
     )
 
     if not word_level:

src/verifier.py

@@ -160,14 +160,13 @@ def validate_citations(response: str, retrieval_result: dict) -> list[str]:
     """
     warnings = []
 
-    # Count unique source documents (not raw chunks)
+    # Count citable units — each retrieved chunk is a separate citable
+    # reference that the LLM may cite as [1], [2], etc.  Counting unique
+    # *files* instead would trigger false "fabricated reference" warnings
+    # when a single file contributes multiple chunks (e.g., a PDF cited
+    # as [1]–[10] for 10 different sections).
     db_results = retrieval_result.get("db_results", [])
-    unique_db_sources = set()
-    for chunk in db_results:
-        source = chunk.get("metadata", {}).get("source", "")
-        if source:
-            unique_db_sources.add(source)
-    db_count = len(unique_db_sources) if unique_db_sources else (1 if db_results else 0)
+    db_count = len(db_results)
     web_count = len(retrieval_result.get("web_results", []))
     sql_count = 1 if retrieval_result.get("sql_results") else 0
     total_sources = db_count + web_count + sql_count
@@ -176,7 +175,7 @@ def validate_citations(response: str, retrieval_result: dict) -> list[str]:
         return warnings
 
     # Extract citation numbers from response BODY only (not References section)
-    refs_split = re.split(r'(?im)^#+\s*(references|sources)\b|^\*\*(references|sources)\*\*', response)
+    refs_split = re.split(r'(?im)^#+\s*(?:references|sources)\s*$|^\*\*(?:references|sources)\*\*\s*$', response)
     body_text = refs_split[0] if refs_split else response
     citation_nums = set(int(m) for m in re.findall(r"\[(\d+)\]", body_text))
     if not citation_nums:
@@ -191,7 +190,7 @@ def validate_citations(response: str, retrieval_result: dict) -> list[str]:
 
     # Check that source file names from retrieval appear in the References section
     refs_match = re.search(
-        r"(?im)(^#+\s*(references|sources)\b|^\*\*(references|sources)\*\*).*",
+        r"(?im)(?:^#+\s*(?:references|sources)\s*$|^\*\*(?:references|sources)\*\*\s*$).*",
         response, re.DOTALL,
     )
     if refs_match:
@@ -204,18 +203,25 @@ def validate_citations(response: str, retrieval_result: dict) -> list[str]:
                 filename = source.rsplit("/", 1)[-1].lower()
                 if filename in refs_text:
                     matched_sources += 1
-        # Also check SQL source files
+        # Also check SQL source — SQL results are plain row dicts (no
+        # metadata wrapper).  All rows come from a single table/source file
+        # whose name is embedded in the context as "Source: <filename>".
+        # Extract it from the combined context instead of iterating rows.
         sql_results = retrieval_result.get("sql_results", [])
-        for sql_chunk in sql_results:
-            sql_source = sql_chunk.get("metadata", {}).get("source", "")
-            if sql_source:
-                sql_filename = sql_source.rsplit("/", 1)[-1].lower()
+        if sql_results:
+            context_text = retrieval_result.get("context", "")
+            sql_source_match = re.search(
+                r"Source:\s*(.+)", context_text,
+            )
+            if sql_source_match:
+                sql_filename = sql_source_match.group(1).strip().rsplit("/", 1)[-1].lower()
                 if sql_filename in refs_text:
                     matched_sources += 1
-        # Also check web source URLs
+        # Also check web source URLs — web results are flat dicts with
+        # a top-level "url" key (no metadata wrapper).
         web_results = retrieval_result.get("web_results", [])
         for web_chunk in web_results:
-            web_url = web_chunk.get("metadata", {}).get("url", "")
+            web_url = web_chunk.get("url", "")
             if web_url and web_url.lower() in refs_text:
                 matched_sources += 1
         # Only warn when db_results are present and are the primary source
@@ -349,7 +355,15 @@ def verify_and_respond(
         user_message = query
 
     # ── Generate initial response ─────────────────────────────────────────
-    response = generate(system_prompt, user_message, cfg, max_tokens=soft_max)
+    try:
+        response = generate(system_prompt, user_message, cfg, max_tokens=soft_max)
+    except Exception as e:
+        return {
+            "response": f"An error occurred while generating the response: {e}",
+            "refused": True,
+            "verification_passed": None,
+            "iterations": 0,
+        }
 
     # ── Guard: empty response from LLM ────────────────────────────────────
     if not response or not response.strip():
@@ -393,19 +407,18 @@ def verify_and_respond(
     max_iterations = verification_cfg.get("max_iterations", 3)
     strict_mode = verification_cfg.get("strict_mode", True)
 
-    # Guard: max_iterations=0 with enabled=true → skip verification
+    # Guard: max_iterations=0 with enabled=true → clamp to 1.
+    # Verification cannot be bypassed via iteration count alone;
+    # users must explicitly set verification.enabled: false.
     if max_iterations <= 0:
-        print("WARNING: Verification iterations set to 0. "
-              "Anti-hallucination verification is effectively disabled.")
-        return {
-            "response": response,
-            "refused": False,
-            "verification_passed": None,
-            "iterations": 0,
-        }
+        print("WARNING: Verification iterations set to 0 but verification "
+              "is enabled. Clamping to 1 iteration. To disable verification, "
+              "set verification.enabled: false.")
+        max_iterations = 1
 
     # ── Verification loop ─────────────────────────────────────────────────
     initial_response = response
+    any_verification_ran = False  # Track whether ANY verification call succeeded
     for iteration in range(1, max_iterations + 1):
         # Layer 5: Warning-phrase scan
         phrase_flags = scan_warning_phrases(response)
@@ -434,6 +447,7 @@ def verify_and_respond(
         except Exception:
             # Verification LLM failed — skip correction, continue to next iteration
             continue
+        any_verification_ran = True
         vr = parse_verification_result(raw_verification)
 
         if vr.get("pass", False):
@@ -448,14 +462,16 @@ def verify_and_respond(
         error_count = vr.get("error_count", len(vr.get("errors", [])))
 
         # Correct and loop for re-verification
+        # Truncate previous response to reduce prompt competition with context
+        truncated_response = response[:1500] + "..." if len(response) > 1500 else response
         correction_prompt = (
             f"The user's original question was: {query}\n\n"
-            f"Your previous response was:\n{response}\n\n"
+            f"Your previous response (truncated for brevity):\n{truncated_response}\n\n"
             f"This response failed verification with "
             f"{error_count} error(s):\n"
             + json.dumps(vr.get("errors", []), indent=2)
-            + "\n\nRewrite your response to fix ALL the issues listed above. "
-            "Use ONLY the provided context. Keep all citation rules."
+            + "\n\nRewrite your COMPLETE response from scratch to fix ALL the issues "
+            "listed above. Use ONLY the provided context. Keep all citation rules."
         )
         try:
             response = generate(
@@ -484,6 +500,7 @@ def verify_and_respond(
                 "You are a strict verification agent. Return only JSON.",
                 vp, cfg, max_tokens=1024,
             ))
+            any_verification_ran = True
         except Exception:
             vr = {"pass": False, "errors": [], "error_count": 0}
         if vr.get("pass", False):
@@ -494,7 +511,39 @@ def verify_and_respond(
                 "iterations": max_iterations,
             }
 
-    # ── Exhausted iterations ──────────────────────────────────────────────
+    # ── Handle total verification system failure ──────────────────────────
+    # If ALL verification LLM calls failed (not "didn't pass" — actually
+    # failed to run), this is a system failure, not a content-quality issue.
+    if not any_verification_ran:
+        print("ERROR: All verification LLM calls failed. "
+              "Verification could not run at all.")
+        if strict_mode:
+            return {
+                "response": (
+                    "Verification was unable to run due to repeated LLM "
+                    "errors. To avoid presenting unverified information, "
+                    "I must decline to answer. Please check your LLM "
+                    "configuration and try again."
+                ),
+                "refused": True,
+                "verification_passed": False,
+                "iterations": max_iterations,
+            }
+        # Non-strict: return with a prominent system-failure warning
+        warning = (
+            "\n\n---\n**WARNING: Verification system failure.** "
+            "The verification pipeline was unable to run due to "
+            "repeated LLM errors. This response has NOT been verified "
+            "at all — treat all claims as unverified."
+        )
+        return {
+            "response": response + warning,
+            "refused": False,
+            "verification_passed": False,
+            "iterations": max_iterations,
+        }
+
+    # ── Exhausted iterations (verification ran but never passed) ──────────
     if strict_mode:
         return {
             "response": REFUSAL_AFTER_VERIFICATION,

tests/test_retriever.py

@@ -203,10 +203,10 @@ def test_retrieve_vector_route_fallback_still_works(
 @patch("src.retriever.retrieve_from_vectordb", return_value=[])
 @patch("src.retriever._run_sql_retrieval")
 @patch("src.retriever._build_fallback_sql_query")
-def test_retrieve_both_route_fallback_still_works(
+def test_retrieve_both_route_fallback_builds_fresh_query(
     mock_build_fallback, mock_run_sql, mock_vectordb, mock_search,
 ):
-    """Existing behavior: route='both', both empty, keyword SQL fallback runs."""
+    """When route='both' and LLM sql_query failed, build a fresh keyword query."""
     from src.retriever import retrieve
 
     # First call: LLM sql_query fails; second call: keyword fallback also fails
@@ -215,10 +215,10 @@ def test_retrieve_both_route_fallback_still_works(
 
     result = retrieve("some data", _BASE_CFG, route="both", sql_query="SELECT * FROM t WHERE x=1")
 
-    # For route="both", sql_query was already set so _build_fallback_sql_query
-    # should NOT be called (existing behavior: reuses the sql_query from QU)
-    mock_build_fallback.assert_not_called()
-    # _run_sql_retrieval called twice: once for LLM query, once for fallback with same query
+    # T2-03 fix: _build_fallback_sql_query IS called to build a fresh keyword
+    # query instead of re-using the already-failed LLM sql_query
+    mock_build_fallback.assert_called_once_with("some data", _BASE_CFG)
+    # _run_sql_retrieval called twice: once for LLM query, once for fresh fallback
     assert mock_run_sql.call_count == 2