LIANJie-Jason Claude Opus 4.6 commited on
Commit
d6a01c7
·
1 Parent(s): 86e20a0

fix: 7 bugs from multi-agent debug review of SQL layer

Browse files

- Column name deduplication after sanitization (Score-A/Score_A collision)
- _safe_str() preserves None in Stata/SPSS/R data loaders
- Reject inf/Infinity in column type inference
- Guard against n=1 in _get_sample_values (ZeroDivisionError)
- NULL values render as "N/A" in SQL context (not "None")
- Defensive .get() in build_schema_summary for malformed schema
- Backtick-quoted table names in _extract_table_from_query regex

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

Files changed (3) hide show
  1. src/retriever.py +1 -1
  2. src/sql_ingest.py +25 -4
  3. src/sql_retriever.py +5 -2
src/retriever.py CHANGED
@@ -139,7 +139,7 @@ def build_combined_context(
139
 
140
  def _extract_table_from_query(sql_query: str) -> str:
141
  """Extract the first table name from a SQL query (best-effort)."""
142
- match = re.search(r'\bFROM\s+"?(\w+)"?', sql_query, re.IGNORECASE)
143
  return match.group(1) if match else ""
144
 
145
 
 
139
 
140
  def _extract_table_from_query(sql_query: str) -> str:
141
  """Extract the first table name from a SQL query (best-effort)."""
142
+ match = re.search(r'\bFROM\s+["`]?(\w+)["`]?', sql_query, re.IGNORECASE)
143
  return match.group(1) if match else ""
144
 
145
 
src/sql_ingest.py CHANGED
@@ -53,7 +53,10 @@ def _infer_column_type(values: list) -> str:
53
  # Try REAL
54
  try:
55
  for v in non_empty:
56
- float(str(v).strip())
 
 
 
57
  return "REAL"
58
  except (ValueError, TypeError):
59
  pass
@@ -84,6 +87,8 @@ def _get_sample_values(values: list, n: int = 5) -> list:
84
  sorted_vals = sorted(unique)
85
  if len(sorted_vals) <= n:
86
  return sorted_vals
 
 
87
  # Pick evenly spaced indices including first and last
88
  indices = [round(i * (len(sorted_vals) - 1) / (n - 1)) for i in range(n)]
89
  return [sorted_vals[i] for i in indices]
@@ -348,12 +353,19 @@ def _load_rows_from_excel(file_path: str, ext: str) -> list[tuple]:
348
  return results
349
 
350
 
 
 
 
 
 
 
 
351
  def _load_rows_from_stata(file_path: str) -> list[tuple]:
352
  """Load Stata .dta into (None, headers, rows) tuples."""
353
  import pyreadstat
354
  df, _meta = pyreadstat.read_dta(file_path)
355
  headers = list(df.columns)
356
- rows = [[str(v) for v in row] for row in df.values.tolist()]
357
  return [(None, headers, rows)]
358
 
359
 
@@ -362,7 +374,7 @@ def _load_rows_from_spss(file_path: str) -> list[tuple]:
362
  import pyreadstat
363
  df, _meta = pyreadstat.read_sav(file_path)
364
  headers = list(df.columns)
365
- rows = [[str(v) for v in row] for row in df.values.tolist()]
366
  return [(None, headers, rows)]
367
 
368
 
@@ -373,7 +385,7 @@ def _load_rows_from_rdata(file_path: str) -> list[tuple]:
373
  tables = []
374
  for name, df in result.items():
375
  headers = list(df.columns)
376
- rows = [[str(v) for v in row] for row in df.values.tolist()]
377
  tables.append((name, headers, rows))
378
  return tables
379
 
@@ -487,6 +499,15 @@ def _ingest_tables(
487
 
488
  # Infer column types, collect samples and stats
489
  safe_headers = [_sanitize_column_name(h) for h in headers]
 
 
 
 
 
 
 
 
 
490
  col_types = []
491
  col_samples = []
492
  col_stats = []
 
53
  # Try REAL
54
  try:
55
  for v in non_empty:
56
+ f = float(str(v).strip())
57
+ # Reject inf/nan — these are not valid REAL data
58
+ if not (-1e308 < f < 1e308):
59
+ raise ValueError("non-finite float")
60
  return "REAL"
61
  except (ValueError, TypeError):
62
  pass
 
87
  sorted_vals = sorted(unique)
88
  if len(sorted_vals) <= n:
89
  return sorted_vals
90
+ if n <= 1:
91
+ return [sorted_vals[0]]
92
  # Pick evenly spaced indices including first and last
93
  indices = [round(i * (len(sorted_vals) - 1) / (n - 1)) for i in range(n)]
94
  return [sorted_vals[i] for i in indices]
 
353
  return results
354
 
355
 
356
+ def _safe_str(v):
357
+ """Convert a value to string, preserving None for missing data."""
358
+ if v is None:
359
+ return None
360
+ return str(v)
361
+
362
+
363
  def _load_rows_from_stata(file_path: str) -> list[tuple]:
364
  """Load Stata .dta into (None, headers, rows) tuples."""
365
  import pyreadstat
366
  df, _meta = pyreadstat.read_dta(file_path)
367
  headers = list(df.columns)
368
+ rows = [[_safe_str(v) for v in row] for row in df.values.tolist()]
369
  return [(None, headers, rows)]
370
 
371
 
 
374
  import pyreadstat
375
  df, _meta = pyreadstat.read_sav(file_path)
376
  headers = list(df.columns)
377
+ rows = [[_safe_str(v) for v in row] for row in df.values.tolist()]
378
  return [(None, headers, rows)]
379
 
380
 
 
385
  tables = []
386
  for name, df in result.items():
387
  headers = list(df.columns)
388
+ rows = [[_safe_str(v) for v in row] for row in df.values.tolist()]
389
  tables.append((name, headers, rows))
390
  return tables
391
 
 
499
 
500
  # Infer column types, collect samples and stats
501
  safe_headers = [_sanitize_column_name(h) for h in headers]
502
+ # Deduplicate: append _2, _3, etc. for collisions
503
+ seen: dict[str, int] = {}
504
+ for i, name in enumerate(safe_headers):
505
+ lower = name.lower()
506
+ if lower in seen:
507
+ seen[lower] += 1
508
+ safe_headers[i] = f"{name}_{seen[lower]}"
509
+ else:
510
+ seen[lower] = 1
511
  col_types = []
512
  col_samples = []
513
  col_stats = []
src/sql_retriever.py CHANGED
@@ -103,7 +103,9 @@ def format_sql_results_as_context(
103
  parts.append("")
104
 
105
  for i, row in enumerate(rows, 1):
106
- fields = ", ".join(f"{k} = {v}" for k, v in row.items())
 
 
107
  parts.append(f"[CHUNK-SQL-{i:03d}] {fields}")
108
  return "\n".join(parts)
109
 
@@ -162,7 +164,8 @@ def build_schema_summary(schema: dict) -> str:
162
  quoted = ", ".join(f'"{s}"' for s in samples[:5])
163
  sample_str = f" e.g. {quoted}"
164
 
165
- line = f" {c['name']} ({meta_str}){sample_str}"
 
166
  if desc:
167
  line += f" \u2014 {desc}"
168
  lines.append(line)
 
103
  parts.append("")
104
 
105
  for i, row in enumerate(rows, 1):
106
+ fields = ", ".join(
107
+ f"{k} = {v if v is not None else 'N/A'}" for k, v in row.items()
108
+ )
109
  parts.append(f"[CHUNK-SQL-{i:03d}] {fields}")
110
  return "\n".join(parts)
111
 
 
164
  quoted = ", ".join(f'"{s}"' for s in samples[:5])
165
  sample_str = f" e.g. {quoted}"
166
 
167
+ col_name = c.get("name", "unknown")
168
+ line = f" {col_name} ({meta_str}){sample_str}"
169
  if desc:
170
  line += f" \u2014 {desc}"
171
  lines.append(line)