Spaces:
Paused
Paused
Commit ·
d6a01c7
1
Parent(s): 86e20a0
fix: 7 bugs from multi-agent debug review of SQL layer
Browse files- Column name deduplication after sanitization (Score-A/Score_A collision)
- _safe_str() preserves None in Stata/SPSS/R data loaders
- Reject inf/Infinity in column type inference
- Guard against n=1 in _get_sample_values (ZeroDivisionError)
- NULL values render as "N/A" in SQL context (not "None")
- Defensive .get() in build_schema_summary for malformed schema
- Backtick-quoted table names in _extract_table_from_query regex
Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
- src/retriever.py +1 -1
- src/sql_ingest.py +25 -4
- src/sql_retriever.py +5 -2
src/retriever.py
CHANGED
|
@@ -139,7 +139,7 @@ def build_combined_context(
|
|
| 139 |
|
| 140 |
def _extract_table_from_query(sql_query: str) -> str:
|
| 141 |
"""Extract the first table name from a SQL query (best-effort)."""
|
| 142 |
-
match = re.search(r'\bFROM\s+"?(\w+)"?', sql_query, re.IGNORECASE)
|
| 143 |
return match.group(1) if match else ""
|
| 144 |
|
| 145 |
|
|
|
|
| 139 |
|
| 140 |
def _extract_table_from_query(sql_query: str) -> str:
|
| 141 |
"""Extract the first table name from a SQL query (best-effort)."""
|
| 142 |
+
match = re.search(r'\bFROM\s+["`]?(\w+)["`]?', sql_query, re.IGNORECASE)
|
| 143 |
return match.group(1) if match else ""
|
| 144 |
|
| 145 |
|
src/sql_ingest.py
CHANGED
|
@@ -53,7 +53,10 @@ def _infer_column_type(values: list) -> str:
|
|
| 53 |
# Try REAL
|
| 54 |
try:
|
| 55 |
for v in non_empty:
|
| 56 |
-
float(str(v).strip())
|
|
|
|
|
|
|
|
|
|
| 57 |
return "REAL"
|
| 58 |
except (ValueError, TypeError):
|
| 59 |
pass
|
|
@@ -84,6 +87,8 @@ def _get_sample_values(values: list, n: int = 5) -> list:
|
|
| 84 |
sorted_vals = sorted(unique)
|
| 85 |
if len(sorted_vals) <= n:
|
| 86 |
return sorted_vals
|
|
|
|
|
|
|
| 87 |
# Pick evenly spaced indices including first and last
|
| 88 |
indices = [round(i * (len(sorted_vals) - 1) / (n - 1)) for i in range(n)]
|
| 89 |
return [sorted_vals[i] for i in indices]
|
|
@@ -348,12 +353,19 @@ def _load_rows_from_excel(file_path: str, ext: str) -> list[tuple]:
|
|
| 348 |
return results
|
| 349 |
|
| 350 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 351 |
def _load_rows_from_stata(file_path: str) -> list[tuple]:
|
| 352 |
"""Load Stata .dta into (None, headers, rows) tuples."""
|
| 353 |
import pyreadstat
|
| 354 |
df, _meta = pyreadstat.read_dta(file_path)
|
| 355 |
headers = list(df.columns)
|
| 356 |
-
rows = [[
|
| 357 |
return [(None, headers, rows)]
|
| 358 |
|
| 359 |
|
|
@@ -362,7 +374,7 @@ def _load_rows_from_spss(file_path: str) -> list[tuple]:
|
|
| 362 |
import pyreadstat
|
| 363 |
df, _meta = pyreadstat.read_sav(file_path)
|
| 364 |
headers = list(df.columns)
|
| 365 |
-
rows = [[
|
| 366 |
return [(None, headers, rows)]
|
| 367 |
|
| 368 |
|
|
@@ -373,7 +385,7 @@ def _load_rows_from_rdata(file_path: str) -> list[tuple]:
|
|
| 373 |
tables = []
|
| 374 |
for name, df in result.items():
|
| 375 |
headers = list(df.columns)
|
| 376 |
-
rows = [[
|
| 377 |
tables.append((name, headers, rows))
|
| 378 |
return tables
|
| 379 |
|
|
@@ -487,6 +499,15 @@ def _ingest_tables(
|
|
| 487 |
|
| 488 |
# Infer column types, collect samples and stats
|
| 489 |
safe_headers = [_sanitize_column_name(h) for h in headers]
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 490 |
col_types = []
|
| 491 |
col_samples = []
|
| 492 |
col_stats = []
|
|
|
|
| 53 |
# Try REAL
|
| 54 |
try:
|
| 55 |
for v in non_empty:
|
| 56 |
+
f = float(str(v).strip())
|
| 57 |
+
# Reject inf/nan — these are not valid REAL data
|
| 58 |
+
if not (-1e308 < f < 1e308):
|
| 59 |
+
raise ValueError("non-finite float")
|
| 60 |
return "REAL"
|
| 61 |
except (ValueError, TypeError):
|
| 62 |
pass
|
|
|
|
| 87 |
sorted_vals = sorted(unique)
|
| 88 |
if len(sorted_vals) <= n:
|
| 89 |
return sorted_vals
|
| 90 |
+
if n <= 1:
|
| 91 |
+
return [sorted_vals[0]]
|
| 92 |
# Pick evenly spaced indices including first and last
|
| 93 |
indices = [round(i * (len(sorted_vals) - 1) / (n - 1)) for i in range(n)]
|
| 94 |
return [sorted_vals[i] for i in indices]
|
|
|
|
| 353 |
return results
|
| 354 |
|
| 355 |
|
| 356 |
+
def _safe_str(v):
|
| 357 |
+
"""Convert a value to string, preserving None for missing data."""
|
| 358 |
+
if v is None:
|
| 359 |
+
return None
|
| 360 |
+
return str(v)
|
| 361 |
+
|
| 362 |
+
|
| 363 |
def _load_rows_from_stata(file_path: str) -> list[tuple]:
|
| 364 |
"""Load Stata .dta into (None, headers, rows) tuples."""
|
| 365 |
import pyreadstat
|
| 366 |
df, _meta = pyreadstat.read_dta(file_path)
|
| 367 |
headers = list(df.columns)
|
| 368 |
+
rows = [[_safe_str(v) for v in row] for row in df.values.tolist()]
|
| 369 |
return [(None, headers, rows)]
|
| 370 |
|
| 371 |
|
|
|
|
| 374 |
import pyreadstat
|
| 375 |
df, _meta = pyreadstat.read_sav(file_path)
|
| 376 |
headers = list(df.columns)
|
| 377 |
+
rows = [[_safe_str(v) for v in row] for row in df.values.tolist()]
|
| 378 |
return [(None, headers, rows)]
|
| 379 |
|
| 380 |
|
|
|
|
| 385 |
tables = []
|
| 386 |
for name, df in result.items():
|
| 387 |
headers = list(df.columns)
|
| 388 |
+
rows = [[_safe_str(v) for v in row] for row in df.values.tolist()]
|
| 389 |
tables.append((name, headers, rows))
|
| 390 |
return tables
|
| 391 |
|
|
|
|
| 499 |
|
| 500 |
# Infer column types, collect samples and stats
|
| 501 |
safe_headers = [_sanitize_column_name(h) for h in headers]
|
| 502 |
+
# Deduplicate: append _2, _3, etc. for collisions
|
| 503 |
+
seen: dict[str, int] = {}
|
| 504 |
+
for i, name in enumerate(safe_headers):
|
| 505 |
+
lower = name.lower()
|
| 506 |
+
if lower in seen:
|
| 507 |
+
seen[lower] += 1
|
| 508 |
+
safe_headers[i] = f"{name}_{seen[lower]}"
|
| 509 |
+
else:
|
| 510 |
+
seen[lower] = 1
|
| 511 |
col_types = []
|
| 512 |
col_samples = []
|
| 513 |
col_stats = []
|
src/sql_retriever.py
CHANGED
|
@@ -103,7 +103,9 @@ def format_sql_results_as_context(
|
|
| 103 |
parts.append("")
|
| 104 |
|
| 105 |
for i, row in enumerate(rows, 1):
|
| 106 |
-
fields = ", ".join(
|
|
|
|
|
|
|
| 107 |
parts.append(f"[CHUNK-SQL-{i:03d}] {fields}")
|
| 108 |
return "\n".join(parts)
|
| 109 |
|
|
@@ -162,7 +164,8 @@ def build_schema_summary(schema: dict) -> str:
|
|
| 162 |
quoted = ", ".join(f'"{s}"' for s in samples[:5])
|
| 163 |
sample_str = f" e.g. {quoted}"
|
| 164 |
|
| 165 |
-
|
|
|
|
| 166 |
if desc:
|
| 167 |
line += f" \u2014 {desc}"
|
| 168 |
lines.append(line)
|
|
|
|
| 103 |
parts.append("")
|
| 104 |
|
| 105 |
for i, row in enumerate(rows, 1):
|
| 106 |
+
fields = ", ".join(
|
| 107 |
+
f"{k} = {v if v is not None else 'N/A'}" for k, v in row.items()
|
| 108 |
+
)
|
| 109 |
parts.append(f"[CHUNK-SQL-{i:03d}] {fields}")
|
| 110 |
return "\n".join(parts)
|
| 111 |
|
|
|
|
| 164 |
quoted = ", ".join(f'"{s}"' for s in samples[:5])
|
| 165 |
sample_str = f" e.g. {quoted}"
|
| 166 |
|
| 167 |
+
col_name = c.get("name", "unknown")
|
| 168 |
+
line = f" {col_name} ({meta_str}){sample_str}"
|
| 169 |
if desc:
|
| 170 |
line += f" \u2014 {desc}"
|
| 171 |
lines.append(line)
|