fix: Round 17 audit — critical/high bug fixes (5 bugs, cross-checked)

- H1: validate_citations regex now matches References:/Reference/Sources (Primary)
- H2: _strip_sql_comments rewritten as state machine, preserves --//**/ inside strings
- H3: SQL citation count fixed: 1 source per query, not 1 per row
- M1: Context budget now accounts for web context size before local allocation
- M2: max_context_chars default aligned to 12000 (was 0 in code vs 12000 in docs)

10 agents dispatched: 5 audit + 3 fix + 2 cross-check. 175/175 tests passing.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

.gitignore

@@ -0,0 +1,16 @@
+.env
+config.yaml
+chroma_db/
+sql_db/
+__pycache__/
+*.pyc
+.DS_Store
+*.egg-info/
+dist/
+build/
+.pytest_cache/
+knowledge_base/*
+!knowledge_base/.gitkeep
+CLAUDE.md
+docs/
+tests/

setup.py

@@ -48,8 +48,9 @@ def generate_config(
         "retrieval": {
             "chunk_size": 1000,
             "chunk_overlap": 100,
-            "top_k": 50,
+            "top_k": 20,
             "max_distance": 0.55,
+            "max_context_chars": 12000,
         },
         "web_search": {
             "enabled": web_search,

src/retriever.py

@@ -43,7 +43,7 @@ def retrieve_from_vectordb(query: str, cfg: dict) -> list[dict]:
     base can still be answered.
     """
     retrieval_cfg = cfg.get("retrieval", {})
-    top_k = retrieval_cfg.get("top_k", 50)
+    top_k = retrieval_cfg.get("top_k", 20)
     max_distance = retrieval_cfg.get("max_distance", 0.55)
     collection = _get_cached_collection(cfg)
 
@@ -93,32 +93,59 @@ def retrieve_from_vectordb(query: str, cfg: dict) -> list[dict]:
     return chunks
 
 
-def format_db_results_as_context(chunks: list[dict]) -> str:
-    """Format vector DB results with CHUNK-LOCAL IDs for citation anchoring."""
+def format_db_results_as_context(chunks: list[dict], max_chars: int = 0) -> str:
+    """Format vector DB results with CHUNK-LOCAL IDs for citation anchoring.
+
+    Args:
+        chunks: Retrieved chunks sorted by distance (most relevant first).
+        max_chars: If > 0, stop adding chunks once total context exceeds
+            this character budget.  Chunks already sorted by relevance
+            ensure the most important content is kept.
+    """
     if not chunks:
         return ""
 
-    parts = ["=== Local Document Results (PRIMARY — always trust these over web sources) ===\n"]
+    header = "=== Local Document Results (PRIMARY — always trust these over web sources) ===\n"
+    parts = [header]
+    total_chars = len(header)
     for i, chunk in enumerate(chunks, 1):
         meta = chunk.get("metadata", {})
         dataset = meta.get("dataset", "")
         dataset_label = f" [Dataset: {dataset}]" if dataset else ""
         source_path = meta.get('source', 'unknown')
-        parts.append(
+        entry = (
             f"[CHUNK-LOCAL-{i:03d}] From: {meta.get('source', 'unknown')}, "
             f"Page/Section {meta.get('page', '?')}{dataset_label}\n"
             f"  Path: {source_path}\n"
             f"  {chunk.get('text', '')}\n"
         )
+        if max_chars > 0 and total_chars + len(entry) > max_chars and i > 1:
+            break
+        parts.append(entry)
+        total_chars += len(entry)
     return "\n".join(parts)
 
 
 def build_combined_context(
     db_results: list[dict], web_results: list[dict], sql_context: str = "",
+    max_context_chars: int = 0,
 ) -> str:
-    """Build combined context string from local, SQL, and web results."""
-    db_context = format_db_results_as_context(db_results)
+    """Build combined context string from local, SQL, and web results.
+
+    When *max_context_chars* > 0, the local-chunk context is truncated to
+    fit within the budget (after accounting for SQL and web context).
+    SQL and web context are never truncated; the budget applies to local chunks.
+    """
+    # SQL context is precise and compact — always include in full.
+    # Web context is supplementary — include in full but account for its size.
+    # Allocate remaining budget to local chunks.
     web_context = format_web_results_as_context(web_results)
+    local_budget = 0
+    if max_context_chars > 0:
+        reserved = len(sql_context) + len(web_context)
+        local_budget = max(max_context_chars - reserved, 2000)
+
+    db_context = format_db_results_as_context(db_results, max_chars=local_budget)
 
     has_local = bool(db_results) or bool(sql_context)
     has_web = bool(web_results)
@@ -428,7 +455,11 @@ def retrieve(
             max_results *= 2
         web_results = search(query, backend=backend, limit=max_results)
 
-    combined_context = build_combined_context(db_results, web_results, sql_context=sql_context)
+    max_context_chars = cfg.get("retrieval", {}).get("max_context_chars", 12000)
+    combined_context = build_combined_context(
+        db_results, web_results, sql_context=sql_context,
+        max_context_chars=max_context_chars,
+    )
     has_sources = bool(db_results) or bool(web_results) or bool(sql_rows)
 
     return {

src/sql_retriever.py

@@ -25,13 +25,47 @@ def _strip_quoted(sql: str) -> str:
 
 
 def _strip_sql_comments(sql: str) -> str:
-    """Remove SQL comments (single-line -- and multi-line /* */) from a query."""
-    prev = None
-    while prev != sql:
-        prev = sql
-        sql = re.sub(r'/\*.*?\*/', '', sql, flags=re.DOTALL)
-        sql = re.sub(r'--[^\n]*', '', sql)
-    return sql
+    """Remove SQL comments (single-line -- and multi-line /* */) from a query.
+
+    Respects single-quoted string literals: ``--`` and ``/* */`` inside
+    quotes are preserved.
+    """
+    result = []
+    i = 0
+    n = len(sql)
+    while i < n:
+        # Single-quoted string literal — copy verbatim (handles '' escaping)
+        if sql[i] == "'":
+            result.append("'")
+            i += 1
+            while i < n:
+                if sql[i] == "'" and i + 1 < n and sql[i + 1] == "'":
+                    result.append("''")
+                    i += 2
+                elif sql[i] == "'":
+                    result.append("'")
+                    i += 1
+                    break
+                else:
+                    result.append(sql[i])
+                    i += 1
+        # Block comment
+        elif sql[i] == '/' and i + 1 < n and sql[i + 1] == '*':
+            i += 2
+            while i < n:
+                if sql[i] == '*' and i + 1 < n and sql[i + 1] == '/':
+                    i += 2
+                    break
+                i += 1
+        # Line comment
+        elif sql[i] == '-' and i + 1 < n and sql[i + 1] == '-':
+            i += 2
+            while i < n and sql[i] != '\n':
+                i += 1
+        else:
+            result.append(sql[i])
+            i += 1
+    return ''.join(result)
 
 
 def _validate_sql(sql: str) -> bool:

src/verifier.py

@@ -168,14 +168,19 @@ def validate_citations(response: str, retrieval_result: dict) -> list[str]:
     db_results = retrieval_result.get("db_results", [])
     db_count = len(db_results)
     web_count = len(retrieval_result.get("web_results", []))
-    sql_count = len(retrieval_result.get("sql_results", []))
+    # SQL rows come from a single table query — count as 1 citable source,
+    # not one per row, since the LLM cites the table as a single reference.
+    sql_count = 1 if retrieval_result.get("sql_results") else 0
     total_sources = db_count + web_count + sql_count
 
     if total_sources == 0:
         return warnings
 
     # Extract citation numbers from response BODY only (not References section)
-    refs_split = re.split(r'(?im)^#+\s*(?:references|sources)\s*$|^\*\*(?:references|sources)\*\*\s*$', response)
+    refs_split = re.split(
+        r'(?im)^#+\s*(?:references?|sources?)[\s:]*(?:\(.*?\))?\s*$|^\*\*(?:references?|sources?)[\s:]*(?:\(.*?\))?\*\*\s*$',
+        response
+    )
     body_text = refs_split[0] if refs_split else response
     citation_nums = set(int(m) for m in re.findall(r"\[(\d+)\]", body_text))
     if not citation_nums:
@@ -190,7 +195,7 @@ def validate_citations(response: str, retrieval_result: dict) -> list[str]:
 
     # Check that source file names from retrieval appear in the References section
     refs_match = re.search(
-        r"(?im)(?:^#+\s*(?:references|sources)\s*$|^\*\*(?:references|sources)\*\*\s*$).*",
+        r"(?im)(?:^#+\s*(?:references?|sources?)[\s:]*(?:\(.*?\))?\s*$|^\*\*(?:references?|sources?)[\s:]*(?:\(.*?\))?\*\*\s*$).*",
         response, re.DOTALL,
     )
     if refs_match: