LIANJie-Jason Claude Opus 4.6 commited on
Commit
a6b3188
·
1 Parent(s): e8de214

fix: Round 15 — 2 HIGH bugs fixed (sql_count citation audit + reader error handling)

Browse files

H-1: validate_citations counted all SQL rows as 1 source, causing
systematic false "fabricated reference" warnings on multi-row SQL
queries. Now counts each row as a separate citable unit.

H-2: text.py and csv_tab.py were the only readers without try/except
wrappers. A single bad .txt/.csv file would crash the entire ingestion
pipeline. Added matching error handling.

174/174 tests passing. Final line-by-line audit by 3 independent agents
confirmed all 23 source files are clean.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

Files changed (3) hide show
  1. src/readers/csv_tab.py +41 -37
  2. src/readers/text.py +8 -4
  3. src/verifier.py +1 -1
src/readers/csv_tab.py CHANGED
@@ -3,47 +3,51 @@ import csv
3
  MAX_CHUNK_CHARS = 6000
4
 
5
  def read_csv_tab(file_path: str, delimiter: str = None) -> list[dict]:
6
- from pathlib import Path
7
- if delimiter is None:
8
- ext = Path(file_path).suffix.lower()
9
- delimiter = "\t" if ext in (".tab", ".tsv") else ","
 
10
 
11
- with open(file_path, "r", encoding="utf-8-sig", errors="replace") as f:
12
- reader = csv.reader(f, delimiter=delimiter)
13
- rows = list(reader)
14
 
15
- if not rows:
16
- return []
 
 
 
 
 
 
 
 
 
 
 
 
17
 
18
- headers = [h.strip().strip('"') for h in rows[0]]
19
- row_texts = []
20
- for row in rows[1:]:
21
- parts = []
22
- padded_row = list(row) + [""] * max(0, len(headers) - len(row))
23
- for header, val in zip(headers, padded_row[:len(headers)]):
24
- val = val.strip().strip('"')
25
- if val:
26
- parts.append(f"{header}: {val}")
27
- if parts:
28
- row_texts.append("; ".join(parts))
29
 
30
- pages = []
31
- header_line = f"Columns: {', '.join(headers)}\n"
32
- block = []
33
- block_chars = len(header_line)
34
- block_start = 1
 
 
 
 
35
 
36
- for idx, row_text in enumerate(row_texts):
37
- if block and block_chars + len(row_text) + 1 > MAX_CHUNK_CHARS:
38
  text = header_line + "\n".join(block)
39
  pages.append({"page": f"rows_{block_start}-{block_start + len(block) - 1}", "text": text})
40
- block = []
41
- block_chars = len(header_line)
42
- block_start = idx + 1
43
- block.append(row_text)
44
- block_chars += len(row_text) + 1
45
-
46
- if block:
47
- text = header_line + "\n".join(block)
48
- pages.append({"page": f"rows_{block_start}-{block_start + len(block) - 1}", "text": text})
49
- return pages
 
3
  MAX_CHUNK_CHARS = 6000
4
 
5
  def read_csv_tab(file_path: str, delimiter: str = None) -> list[dict]:
6
+ try:
7
+ from pathlib import Path
8
+ if delimiter is None:
9
+ ext = Path(file_path).suffix.lower()
10
+ delimiter = "\t" if ext in (".tab", ".tsv") else ","
11
 
12
+ with open(file_path, "r", encoding="utf-8-sig", errors="replace") as f:
13
+ reader = csv.reader(f, delimiter=delimiter)
14
+ rows = list(reader)
15
 
16
+ if not rows:
17
+ return []
18
+
19
+ headers = [h.strip().strip('"') for h in rows[0]]
20
+ row_texts = []
21
+ for row in rows[1:]:
22
+ parts = []
23
+ padded_row = list(row) + [""] * max(0, len(headers) - len(row))
24
+ for header, val in zip(headers, padded_row[:len(headers)]):
25
+ val = val.strip().strip('"')
26
+ if val:
27
+ parts.append(f"{header}: {val}")
28
+ if parts:
29
+ row_texts.append("; ".join(parts))
30
 
31
+ pages = []
32
+ header_line = f"Columns: {', '.join(headers)}\n"
33
+ block = []
34
+ block_chars = len(header_line)
35
+ block_start = 1
 
 
 
 
 
 
36
 
37
+ for idx, row_text in enumerate(row_texts):
38
+ if block and block_chars + len(row_text) + 1 > MAX_CHUNK_CHARS:
39
+ text = header_line + "\n".join(block)
40
+ pages.append({"page": f"rows_{block_start}-{block_start + len(block) - 1}", "text": text})
41
+ block = []
42
+ block_chars = len(header_line)
43
+ block_start = idx + 1
44
+ block.append(row_text)
45
+ block_chars += len(row_text) + 1
46
 
47
+ if block:
 
48
  text = header_line + "\n".join(block)
49
  pages.append({"page": f"rows_{block_start}-{block_start + len(block) - 1}", "text": text})
50
+ return pages
51
+ except Exception as e:
52
+ print(f"Warning: Could not read {file_path}: {e}")
53
+ return []
 
 
 
 
 
 
src/readers/text.py CHANGED
@@ -2,8 +2,12 @@
2
 
3
 
4
  def read_text(file_path: str) -> list[dict]:
5
- with open(file_path, "r", encoding="utf-8", errors="replace") as f:
6
- text = f.read()
7
- if not text.strip():
 
 
 
 
 
8
  return []
9
- return [{"page": 1, "text": text.strip()}]
 
2
 
3
 
4
  def read_text(file_path: str) -> list[dict]:
5
+ try:
6
+ with open(file_path, "r", encoding="utf-8", errors="replace") as f:
7
+ text = f.read()
8
+ if not text.strip():
9
+ return []
10
+ return [{"page": 1, "text": text.strip()}]
11
+ except Exception as e:
12
+ print(f"Warning: Could not read {file_path}: {e}")
13
  return []
 
src/verifier.py CHANGED
@@ -168,7 +168,7 @@ def validate_citations(response: str, retrieval_result: dict) -> list[str]:
168
  db_results = retrieval_result.get("db_results", [])
169
  db_count = len(db_results)
170
  web_count = len(retrieval_result.get("web_results", []))
171
- sql_count = 1 if retrieval_result.get("sql_results") else 0
172
  total_sources = db_count + web_count + sql_count
173
 
174
  if total_sources == 0:
 
168
  db_results = retrieval_result.get("db_results", [])
169
  db_count = len(db_results)
170
  web_count = len(retrieval_result.get("web_results", []))
171
+ sql_count = len(retrieval_result.get("sql_results", []))
172
  total_sources = db_count + web_count + sql_count
173
 
174
  if total_sources == 0: