LIANJie-Jason Claude Opus 4.6 commited on
Commit
91410f8
·
1 Parent(s): 89e2c52

fix: pre-release audit round 9 — anti-hallucination hardening + error handling

Browse files

CRITICAL: Correction prompt now includes previous response for targeted
fixing instead of blind regeneration (AH-06).

HIGH fixes:
- Add deterministic citation audit Layer 4.5 (AH-12)
- Strengthen verification to 10-point checklist with exact-evidence
requirement (AH-07)
- Warn when verification disabled or max_iterations=0 (AH-08)
- Fix Gemini NameError crash in ValueError handler (ERR-1)
- Add error handling to OpenAI/Anthropic with sanitized messages (ERR-2)
- Clear stale data when knowledge_base/ is empty (BUG-P1)

MEDIUM fixes:
- Sanitize Gemini exception messages to prevent API key leaks (SEC-1)
- Add config.yaml to .gitignore (SEC-2)
- Detect Gemini error strings in verifier guard
- Recompute citation warnings each verification iteration

174/174 tests passing. Bug fix log updated.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

.gitignore CHANGED
@@ -1,4 +1,5 @@
1
  .env
 
2
  chroma_db/
3
  sql_db/
4
  __pycache__/
 
1
  .env
2
+ config.yaml
3
  chroma_db/
4
  sql_db/
5
  __pycache__/
src/ingest.py CHANGED
@@ -161,6 +161,47 @@ def ingest_documents(cfg: dict = None, documents_dir: str = None) -> int:
161
  if not files:
162
  print(f"No supported files found in {documents_dir}")
163
  print(f"Supported types: {', '.join(sorted(READERS.keys()))}")
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
164
  return 0
165
 
166
  # Summarize
 
161
  if not files:
162
  print(f"No supported files found in {documents_dir}")
163
  print(f"Supported types: {', '.join(sorted(READERS.keys()))}")
164
+ # Clear stale data from previous ingestions
165
+ try:
166
+ collection = get_chroma_collection(cfg)
167
+ existing = collection.count()
168
+ if existing > 0:
169
+ print(f"Clearing {existing} stale chunks from previous ingestion...")
170
+ all_ids = collection.get().get("ids", [])
171
+ if all_ids:
172
+ for ci in range(0, len(all_ids), 5000):
173
+ collection.delete(ids=all_ids[ci:ci + 5000])
174
+ except Exception as e:
175
+ print(f"Warning: Could not clear vector DB: {e}")
176
+ # Clear SQL database
177
+ sql_enabled = cfg.get("sql", {}).get("enabled", True)
178
+ if sql_enabled:
179
+ try:
180
+ sql_db_dir = cfg.get("paths", {}).get("sql_db", "sql_db")
181
+ if not os.path.isabs(sql_db_dir):
182
+ project_root = Path(__file__).resolve().parent.parent
183
+ sql_db_dir = os.path.join(str(project_root), sql_db_dir)
184
+ db_file = os.path.join(sql_db_dir, "knowledge_base.db")
185
+ schema_file = os.path.join(sql_db_dir, "sql_schemas.json")
186
+ if os.path.exists(db_file):
187
+ os.remove(db_file)
188
+ print("Cleared stale SQL database.")
189
+ if os.path.exists(schema_file):
190
+ os.remove(schema_file)
191
+ except Exception as e:
192
+ print(f"Warning: Could not clear SQL database: {e}")
193
+ # Clear KB meta
194
+ try:
195
+ db_path = cfg.get("paths", {}).get("vector_db", "chroma_db")
196
+ if not os.path.isabs(db_path):
197
+ project_root = Path(__file__).resolve().parent.parent
198
+ db_path = os.path.join(str(project_root), db_path)
199
+ meta_file = os.path.join(db_path, "kb_meta.txt")
200
+ if os.path.exists(meta_file):
201
+ os.remove(meta_file)
202
+ print("Cleared stale KB overview.")
203
+ except Exception as e:
204
+ print(f"Warning: Could not clear KB meta: {e}")
205
  return 0
206
 
207
  # Summarize
src/llm/anthropic.py CHANGED
@@ -7,14 +7,18 @@ def generate(system_prompt: str, user_message: str, api_key: str,
7
  model = model or "claude-sonnet-4-6"
8
  from anthropic import Anthropic
9
  client = Anthropic(api_key=api_key)
10
- response = client.messages.create(
11
- model=model, max_tokens=max_tokens, temperature=temperature,
12
- system=system_prompt,
13
- messages=[{"role": "user", "content": user_message}],
14
- )
15
- if not response.content:
16
- return ""
17
- return response.content[0].text or ""
 
 
 
 
18
 
19
  def list_models(api_key: str) -> list[str]:
20
  try:
 
7
  model = model or "claude-sonnet-4-6"
8
  from anthropic import Anthropic
9
  client = Anthropic(api_key=api_key)
10
+ try:
11
+ response = client.messages.create(
12
+ model=model, max_tokens=max_tokens, temperature=temperature,
13
+ system=system_prompt,
14
+ messages=[{"role": "user", "content": user_message}],
15
+ )
16
+ if not response.content:
17
+ return ""
18
+ return response.content[0].text or ""
19
+ except Exception as e:
20
+ error_type = type(e).__name__
21
+ raise RuntimeError(f"Anthropic API error ({error_type}). Check your API key and network connection.") from e
22
 
23
  def list_models(api_key: str) -> list[str]:
24
  try:
src/llm/gemini.py CHANGED
@@ -11,17 +11,19 @@ def generate(system_prompt: str, user_message: str, api_key: str,
11
  model_name=model, system_instruction=system_prompt,
12
  generation_config=genai.GenerationConfig(temperature=temperature, max_output_tokens=max_tokens),
13
  )
 
14
  try:
15
  response = gen_model.generate_content(user_message)
16
  return response.text
17
  except ValueError:
18
  # Gemini raises ValueError when the response is blocked by safety filters.
19
- block_reason = getattr(response, "prompt_feedback", None)
20
- if block_reason:
21
- return f"[Gemini blocked: {block_reason}]"
 
22
  return "[Gemini blocked: response filtered by safety settings]"
23
- except Exception as e:
24
- return f"[Gemini error: {e}]"
25
 
26
  def list_models(api_key: str) -> list[str]:
27
  try:
 
11
  model_name=model, system_instruction=system_prompt,
12
  generation_config=genai.GenerationConfig(temperature=temperature, max_output_tokens=max_tokens),
13
  )
14
+ response = None
15
  try:
16
  response = gen_model.generate_content(user_message)
17
  return response.text
18
  except ValueError:
19
  # Gemini raises ValueError when the response is blocked by safety filters.
20
+ if response is not None:
21
+ block_reason = getattr(response, "prompt_feedback", None)
22
+ if block_reason:
23
+ return f"[Gemini blocked: {block_reason}]"
24
  return "[Gemini blocked: response filtered by safety settings]"
25
+ except Exception:
26
+ return "[Gemini error: request failed]"
27
 
28
  def list_models(api_key: str) -> list[str]:
29
  try:
src/llm/openai.py CHANGED
@@ -8,30 +8,34 @@ def generate(system_prompt: str, user_message: str, api_key: str,
8
  from openai import OpenAI
9
  client = OpenAI(api_key=api_key)
10
  is_reasoning = model.startswith(("o1", "o3", "o4"))
11
- if is_reasoning:
12
- messages = [
13
- {"role": "developer", "content": system_prompt},
14
- {"role": "user", "content": user_message},
15
- ]
16
- response = client.chat.completions.create(
17
- model=model,
18
- messages=messages,
19
- max_completion_tokens=max_tokens,
20
- )
21
- else:
22
- messages = [
23
- {"role": "system", "content": system_prompt},
24
- {"role": "user", "content": user_message},
25
- ]
26
- response = client.chat.completions.create(
27
- model=model,
28
- messages=messages,
29
- temperature=temperature,
30
- max_tokens=max_tokens,
31
- )
32
- if not response.choices:
33
- return ""
34
- return response.choices[0].message.content or ""
 
 
 
 
35
 
36
  def list_models(api_key: str) -> list[str]:
37
  try:
 
8
  from openai import OpenAI
9
  client = OpenAI(api_key=api_key)
10
  is_reasoning = model.startswith(("o1", "o3", "o4"))
11
+ try:
12
+ if is_reasoning:
13
+ messages = [
14
+ {"role": "developer", "content": system_prompt},
15
+ {"role": "user", "content": user_message},
16
+ ]
17
+ response = client.chat.completions.create(
18
+ model=model,
19
+ messages=messages,
20
+ max_completion_tokens=max_tokens,
21
+ )
22
+ else:
23
+ messages = [
24
+ {"role": "system", "content": system_prompt},
25
+ {"role": "user", "content": user_message},
26
+ ]
27
+ response = client.chat.completions.create(
28
+ model=model,
29
+ messages=messages,
30
+ temperature=temperature,
31
+ max_tokens=max_tokens,
32
+ )
33
+ if not response.choices:
34
+ return ""
35
+ return response.choices[0].message.content or ""
36
+ except Exception as e:
37
+ error_type = type(e).__name__
38
+ raise RuntimeError(f"OpenAI API error ({error_type}). Check your API key and network connection.") from e
39
 
40
  def list_models(api_key: str) -> list[str]:
41
  try:
src/prompts.py CHANGED
@@ -98,10 +98,10 @@ problems.
98
  {similarity_flags}
99
  --- END SIMILARITY FLAGS ---
100
 
101
- Run through this 9-point verification checklist and report results:
102
 
103
  1. Does every factual claim have a numbered citation endnote (e.g. [1], [2])?
104
- 2. Are all citations grounded in the provided context?
105
  3. Are there any claims that appear fabricated or hallucinated?
106
  4. Are direct quotes accurately reproduced from the context?
107
  5. Does the References section exist and list all cited sources?
@@ -109,6 +109,7 @@ Run through this 9-point verification checklist and report results:
109
  7. Are any phrase-level flags confirmed as hallucinations?
110
  8. Do similarity flags indicate unsupported semantic drift?
111
  9. Is the response length proportional to the available evidence?
 
112
 
113
  Return your analysis as JSON in the following format:
114
 
 
98
  {similarity_flags}
99
  --- END SIMILARITY FLAGS ---
100
 
101
+ Run through this 10-point verification checklist and report results:
102
 
103
  1. Does every factual claim have a numbered citation endnote (e.g. [1], [2])?
104
+ 2. For each cited claim, can you find the EXACT sentence or data point in the context that supports it? If you cannot point to specific supporting text, flag it as an error.
105
  3. Are there any claims that appear fabricated or hallucinated?
106
  4. Are direct quotes accurately reproduced from the context?
107
  5. Does the References section exist and list all cited sources?
 
109
  7. Are any phrase-level flags confirmed as hallucinations?
110
  8. Do similarity flags indicate unsupported semantic drift?
111
  9. Is the response length proportional to the available evidence?
112
+ 10. Do the citation numbers match the correct sources? (e.g., [1] must refer to the first source listed in References, and that source must actually support the claim)
113
 
114
  Return your analysis as JSON in the following format:
115
 
src/verifier.py CHANGED
@@ -148,6 +148,60 @@ def compute_similarity_flags(
148
  return flags
149
 
150
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
151
  # ── Layer 3 helper: parse LLM verification JSON ──────────────────────────────
152
 
153
  def parse_verification_result(raw: str) -> dict:
@@ -269,7 +323,9 @@ def verify_and_respond(
269
 
270
  # ── Guard: detect provider-level content blocks ────────────────────────
271
  lower_resp = response.lower()
272
- if lower_resp.startswith("[gemini blocked") or lower_resp.startswith("[blocked"):
 
 
273
  return {
274
  "response": (
275
  "The LLM provider blocked this request due to content "
@@ -280,9 +336,14 @@ def verify_and_respond(
280
  "iterations": 0,
281
  }
282
 
 
 
 
283
  # ── Short-circuit if verification is disabled ─────────────────────────
284
  verification_cfg = cfg.get("verification", {})
285
  if not verification_cfg.get("enabled", True):
 
 
286
  return {
287
  "response": response,
288
  "refused": False,
@@ -295,6 +356,8 @@ def verify_and_respond(
295
 
296
  # Guard: max_iterations=0 with enabled=true → skip verification
297
  if max_iterations <= 0:
 
 
298
  return {
299
  "response": response,
300
  "refused": False,
@@ -313,9 +376,14 @@ def verify_and_respond(
313
  sim_flags = compute_similarity_flags(response, context, cfg)
314
  sim_flag_strs = [f["message"] for f in sim_flags]
315
 
 
 
 
 
316
  # Layer 3: LLM-as-verifier
 
317
  verification_prompt = build_verification_prompt(
318
- response, context, phrase_flag_strs, sim_flag_strs,
319
  )
320
  try:
321
  raw_verification = generate(
@@ -342,10 +410,11 @@ def verify_and_respond(
342
  # Correct and loop for re-verification
343
  correction_prompt = (
344
  f"The user's original question was: {query}\n\n"
345
- f"Your previous response failed verification with "
 
346
  f"{error_count} error(s):\n"
347
  + json.dumps(vr.get("errors", []), indent=2)
348
- + "\n\nRewrite your response to fix ALL issues. "
349
  "Use ONLY the provided context. Keep all citation rules."
350
  )
351
  try:
@@ -363,10 +432,12 @@ def verify_and_respond(
363
  if response != initial_response:
364
  phrase_flags = scan_warning_phrases(response)
365
  similarity_flags = compute_similarity_flags(response, context, cfg)
 
 
366
  vp = build_verification_prompt(
367
  response, context,
368
  [f["message"] for f in phrase_flags],
369
- [f["message"] for f in similarity_flags],
370
  )
371
  try:
372
  vr = parse_verification_result(generate(
@@ -398,6 +469,8 @@ def verify_and_respond(
398
  "against the provided sources. Some claims may lack adequate "
399
  "grounding. Please cross-check important facts."
400
  )
 
 
401
  return {
402
  "response": response + warning,
403
  "refused": False,
 
148
  return flags
149
 
150
 
151
+ def validate_citations(response: str, retrieval_result: dict) -> list[str]:
152
+ """Deterministic citation audit (Layer 4.5).
153
+
154
+ Checks that:
155
+ - Citation numbers [N] in the response body don't exceed source count.
156
+ - References section lists source names that match actual retrieved sources.
157
+
158
+ Returns a list of warning strings (empty if all checks pass).
159
+ """
160
+ warnings = []
161
+
162
+ # Count actual sources available
163
+ db_count = len(retrieval_result.get("db_results", []))
164
+ web_count = len(retrieval_result.get("web_results", []))
165
+ sql_count = len(retrieval_result.get("sql_results", []))
166
+ total_sources = db_count + web_count + sql_count
167
+
168
+ if total_sources == 0:
169
+ return warnings
170
+
171
+ # Extract citation numbers from response body
172
+ citation_nums = set(int(m) for m in re.findall(r"\[(\d+)\]", response))
173
+ if not citation_nums:
174
+ return warnings
175
+
176
+ max_citation = max(citation_nums)
177
+ if max_citation > total_sources:
178
+ warnings.append(
179
+ f"Citation [{max_citation}] exceeds available source count "
180
+ f"({total_sources}). Possible fabricated reference."
181
+ )
182
+
183
+ # Check that source file names from retrieval appear in the References section
184
+ refs_match = re.search(r"(?i)\b(references|sources)\b.*", response, re.DOTALL)
185
+ if refs_match:
186
+ refs_text = refs_match.group(0).lower()
187
+ db_results = retrieval_result.get("db_results", [])
188
+ matched_sources = 0
189
+ for chunk in db_results:
190
+ source = chunk.get("metadata", {}).get("source", "")
191
+ if source:
192
+ # Check for filename (last component of path)
193
+ filename = source.rsplit("/", 1)[-1].lower()
194
+ if filename in refs_text:
195
+ matched_sources += 1
196
+ if db_results and matched_sources == 0:
197
+ warnings.append(
198
+ "References section does not mention any filenames from "
199
+ "the retrieved local sources. Citations may be fabricated."
200
+ )
201
+
202
+ return warnings
203
+
204
+
205
  # ── Layer 3 helper: parse LLM verification JSON ──────────────────────────────
206
 
207
  def parse_verification_result(raw: str) -> dict:
 
323
 
324
  # ── Guard: detect provider-level content blocks ────────────────────────
325
  lower_resp = response.lower()
326
+ if (lower_resp.startswith("[gemini blocked")
327
+ or lower_resp.startswith("[gemini error")
328
+ or lower_resp.startswith("[blocked")):
329
  return {
330
  "response": (
331
  "The LLM provider blocked this request due to content "
 
336
  "iterations": 0,
337
  }
338
 
339
+ # ── Layer 4.5: Deterministic citation audit ───────────────────────
340
+ citation_warnings = validate_citations(response, retrieval_result)
341
+
342
  # ── Short-circuit if verification is disabled ─────────────────────────
343
  verification_cfg = cfg.get("verification", {})
344
  if not verification_cfg.get("enabled", True):
345
+ print("WARNING: Verification is disabled (verification.enabled=false). "
346
+ "Responses may contain ungrounded claims.")
347
  return {
348
  "response": response,
349
  "refused": False,
 
356
 
357
  # Guard: max_iterations=0 with enabled=true → skip verification
358
  if max_iterations <= 0:
359
+ print("WARNING: Verification iterations set to 0. "
360
+ "Anti-hallucination verification is effectively disabled.")
361
  return {
362
  "response": response,
363
  "refused": False,
 
376
  sim_flags = compute_similarity_flags(response, context, cfg)
377
  sim_flag_strs = [f["message"] for f in sim_flags]
378
 
379
+ # Layer 4.5: Citation audit (recomputed each iteration after corrections)
380
+ citation_warnings = validate_citations(response, retrieval_result)
381
+ citation_flag_strs = citation_warnings if citation_warnings else []
382
+
383
  # Layer 3: LLM-as-verifier
384
+ all_sim_flags = sim_flag_strs + citation_flag_strs
385
  verification_prompt = build_verification_prompt(
386
+ response, context, phrase_flag_strs, all_sim_flags,
387
  )
388
  try:
389
  raw_verification = generate(
 
410
  # Correct and loop for re-verification
411
  correction_prompt = (
412
  f"The user's original question was: {query}\n\n"
413
+ f"Your previous response was:\n{response}\n\n"
414
+ f"This response failed verification with "
415
  f"{error_count} error(s):\n"
416
  + json.dumps(vr.get("errors", []), indent=2)
417
+ + "\n\nRewrite your response to fix ALL the issues listed above. "
418
  "Use ONLY the provided context. Keep all citation rules."
419
  )
420
  try:
 
432
  if response != initial_response:
433
  phrase_flags = scan_warning_phrases(response)
434
  similarity_flags = compute_similarity_flags(response, context, cfg)
435
+ citation_flags_final = validate_citations(response, retrieval_result)
436
+ all_sim_final = [f["message"] for f in similarity_flags] + citation_flags_final
437
  vp = build_verification_prompt(
438
  response, context,
439
  [f["message"] for f in phrase_flags],
440
+ all_sim_final,
441
  )
442
  try:
443
  vr = parse_verification_result(generate(
 
469
  "against the provided sources. Some claims may lack adequate "
470
  "grounding. Please cross-check important facts."
471
  )
472
+ if citation_warnings:
473
+ warning += "\n**Citation issues:** " + "; ".join(citation_warnings)
474
  return {
475
  "response": response + warning,
476
  "refused": False,