Spaces:
Sleeping
Sleeping
Commit ·
3d1d5b4
1
Parent(s): 353f65a
feat: add LLM-powered column descriptions with codebook detection
Browse filesDuring SQL ingestion, each table columns are now described by the LLM:
1. Search same directory for codebook/documentation files (PDF, DOCX, TXT, MD)
2. If codebook found, pass its text + column metadata to LLM for descriptions
3. If no codebook, LLM infers column meanings from names, types, samples
4. Descriptions stored in sql_schemas.json alongside existing column metadata
Column descriptions are always visible to the chatbot:
- build_schema_summary() includes descriptions in the QU prompt
- format_sql_results_as_context() includes descriptions with SQL results
- Graceful degradation: LLM failure is non-fatal
Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
- src/retriever.py +4 -1
- src/sql_ingest.py +257 -8
- src/sql_retriever.py +57 -11
src/retriever.py
CHANGED
|
@@ -171,7 +171,10 @@ def _run_sql_retrieval(sql_query: str, cfg: dict) -> tuple[list[dict], str]:
|
|
| 171 |
pass
|
| 172 |
table_name = _extract_table_from_query(sql_query)
|
| 173 |
source_file = _lookup_source_file(table_name, schema)
|
| 174 |
-
|
|
|
|
|
|
|
|
|
|
| 175 |
return sql_rows, sql_context
|
| 176 |
|
| 177 |
|
|
|
|
| 171 |
pass
|
| 172 |
table_name = _extract_table_from_query(sql_query)
|
| 173 |
source_file = _lookup_source_file(table_name, schema)
|
| 174 |
+
table_info = schema.get(table_name)
|
| 175 |
+
sql_context = format_sql_results_as_context(
|
| 176 |
+
sql_rows, sql_query, source_file, table_info=table_info,
|
| 177 |
+
)
|
| 178 |
return sql_rows, sql_context
|
| 179 |
|
| 180 |
|
src/sql_ingest.py
CHANGED
|
@@ -10,6 +10,15 @@ from pathlib import Path
|
|
| 10 |
# Extensions that trigger SQL ingestion (tabular formats)
|
| 11 |
SQL_EXTENSIONS = {".csv", ".tab", ".tsv", ".xlsx", ".xls", ".dta", ".sav", ".rds", ".rda"}
|
| 12 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 13 |
|
| 14 |
def _sanitize_part(s: str) -> str:
|
| 15 |
"""Sanitize a single name part: replace non-alnum with _, collapse, strip."""
|
|
@@ -51,19 +60,246 @@ def _infer_column_type(values: list) -> str:
|
|
| 51 |
return "TEXT"
|
| 52 |
|
| 53 |
|
| 54 |
-
def _get_sample_values(values: list, n: int =
|
| 55 |
-
"""Get up to n unique non-empty sample values.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 56 |
seen = set()
|
| 57 |
-
samples = []
|
| 58 |
for v in values:
|
| 59 |
if v is not None and str(v).strip() and str(v).lower() != "nan":
|
| 60 |
v_str = str(v).strip()
|
| 61 |
if v_str not in seen:
|
| 62 |
seen.add(v_str)
|
| 63 |
-
|
| 64 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 65 |
break
|
| 66 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 67 |
|
| 68 |
|
| 69 |
def _load_rows_from_csv(file_path: str, ext: str) -> list[tuple]:
|
|
@@ -204,6 +440,15 @@ def ingest_to_sql(files: list[tuple], documents_dir: str, cfg: dict) -> dict:
|
|
| 204 |
finally:
|
| 205 |
conn.close()
|
| 206 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 207 |
with open(schema_path, "w", encoding="utf-8") as f:
|
| 208 |
json.dump(schema_registry, f, indent=2, default=str)
|
| 209 |
|
|
@@ -232,14 +477,17 @@ def _ingest_tables(
|
|
| 232 |
for sheet_or_name, headers, rows in tables:
|
| 233 |
table_name = _sanitize_table_name(dataset_name, file_path.stem, ext, sheet_or_name)
|
| 234 |
|
| 235 |
-
# Infer column types
|
| 236 |
safe_headers = [_sanitize_column_name(h) for h in headers]
|
| 237 |
col_types = []
|
| 238 |
col_samples = []
|
|
|
|
| 239 |
for col_idx in range(len(headers)):
|
| 240 |
col_values = [row[col_idx] if col_idx < len(row) else None for row in rows]
|
| 241 |
-
|
|
|
|
| 242 |
col_samples.append(_get_sample_values(col_values))
|
|
|
|
| 243 |
|
| 244 |
# Create table
|
| 245 |
col_defs = ", ".join(f'"{h}" {t}' for h, t in zip(safe_headers, col_types))
|
|
@@ -282,6 +530,7 @@ def _ingest_tables(
|
|
| 282 |
"original_name": headers[i],
|
| 283 |
"type": col_types[i],
|
| 284 |
"sample": col_samples[i],
|
|
|
|
| 285 |
}
|
| 286 |
for i in range(len(headers))
|
| 287 |
],
|
|
|
|
| 10 |
# Extensions that trigger SQL ingestion (tabular formats)
|
| 11 |
SQL_EXTENSIONS = {".csv", ".tab", ".tsv", ".xlsx", ".xls", ".dta", ".sav", ".rds", ".rda"}
|
| 12 |
|
| 13 |
+
# Extensions that might contain codebook/documentation
|
| 14 |
+
_CODEBOOK_EXTENSIONS = {".pdf", ".docx", ".txt", ".md"}
|
| 15 |
+
|
| 16 |
+
# Keywords suggesting a file is a codebook
|
| 17 |
+
_CODEBOOK_KEYWORDS = {
|
| 18 |
+
"codebook", "readme", "dictionary", "documentation",
|
| 19 |
+
"manual", "guide", "metadata", "variable", "description",
|
| 20 |
+
}
|
| 21 |
+
|
| 22 |
|
| 23 |
def _sanitize_part(s: str) -> str:
|
| 24 |
"""Sanitize a single name part: replace non-alnum with _, collapse, strip."""
|
|
|
|
| 60 |
return "TEXT"
|
| 61 |
|
| 62 |
|
| 63 |
+
def _get_sample_values(values: list, n: int = 5) -> list:
|
| 64 |
+
"""Get a representative spread of up to n unique non-empty sample values.
|
| 65 |
+
|
| 66 |
+
Instead of taking the first n values (which may be biased by sort order),
|
| 67 |
+
picks evenly spaced values from the sorted unique set so the LLM sees
|
| 68 |
+
the full range (e.g., first, middle, last country names).
|
| 69 |
+
"""
|
| 70 |
+
unique = []
|
| 71 |
seen = set()
|
|
|
|
| 72 |
for v in values:
|
| 73 |
if v is not None and str(v).strip() and str(v).lower() != "nan":
|
| 74 |
v_str = str(v).strip()
|
| 75 |
if v_str not in seen:
|
| 76 |
seen.add(v_str)
|
| 77 |
+
unique.append(v_str)
|
| 78 |
+
if not unique:
|
| 79 |
+
return []
|
| 80 |
+
sorted_vals = sorted(unique)
|
| 81 |
+
if len(sorted_vals) <= n:
|
| 82 |
+
return sorted_vals
|
| 83 |
+
# Pick evenly spaced indices including first and last
|
| 84 |
+
indices = [round(i * (len(sorted_vals) - 1) / (n - 1)) for i in range(n)]
|
| 85 |
+
return [sorted_vals[i] for i in indices]
|
| 86 |
+
|
| 87 |
+
|
| 88 |
+
def _get_column_stats(values: list, col_type: str) -> dict:
|
| 89 |
+
"""Compute column statistics for schema metadata.
|
| 90 |
+
|
| 91 |
+
Returns dict with 'unique_count' and optionally 'min'/'max' for numerics.
|
| 92 |
+
"""
|
| 93 |
+
non_empty = [
|
| 94 |
+
str(v).strip() for v in values
|
| 95 |
+
if v is not None and str(v).strip() and str(v).lower() != "nan"
|
| 96 |
+
]
|
| 97 |
+
unique_count = len(set(non_empty))
|
| 98 |
+
stats = {"unique_count": unique_count}
|
| 99 |
+
if col_type == "INTEGER" and non_empty:
|
| 100 |
+
try:
|
| 101 |
+
nums = [int(v) for v in non_empty]
|
| 102 |
+
stats["min"] = min(nums)
|
| 103 |
+
stats["max"] = max(nums)
|
| 104 |
+
except (ValueError, TypeError):
|
| 105 |
+
pass
|
| 106 |
+
elif col_type == "REAL" and non_empty:
|
| 107 |
+
try:
|
| 108 |
+
nums = [float(v) for v in non_empty]
|
| 109 |
+
stats["min"] = min(nums)
|
| 110 |
+
stats["max"] = max(nums)
|
| 111 |
+
except (ValueError, TypeError):
|
| 112 |
+
pass
|
| 113 |
+
return stats
|
| 114 |
+
|
| 115 |
+
|
| 116 |
+
def _find_codebook_files(tabular_file: Path) -> list[Path]:
|
| 117 |
+
"""Search for codebook/documentation files in the same directory as the tabular file."""
|
| 118 |
+
parent = tabular_file.parent
|
| 119 |
+
candidates = []
|
| 120 |
+
for f in sorted(parent.iterdir()):
|
| 121 |
+
if not f.is_file():
|
| 122 |
+
continue
|
| 123 |
+
if f.suffix.lower() not in _CODEBOOK_EXTENSIONS:
|
| 124 |
+
continue
|
| 125 |
+
candidates.append(f)
|
| 126 |
+
|
| 127 |
+
# Sort: files with codebook keywords in name come first
|
| 128 |
+
def priority(p):
|
| 129 |
+
name_lower = p.stem.lower()
|
| 130 |
+
return 0 if any(kw in name_lower for kw in _CODEBOOK_KEYWORDS) else 1
|
| 131 |
+
|
| 132 |
+
candidates.sort(key=priority)
|
| 133 |
+
return candidates
|
| 134 |
+
|
| 135 |
+
|
| 136 |
+
def _read_codebook_text(codebook_files: list[Path], max_chars: int = 4000) -> str:
|
| 137 |
+
"""Read codebook files and return concatenated text, truncated."""
|
| 138 |
+
from src.readers import read_file
|
| 139 |
+
|
| 140 |
+
texts = []
|
| 141 |
+
total = 0
|
| 142 |
+
for f in codebook_files:
|
| 143 |
+
try:
|
| 144 |
+
pages = read_file(str(f))
|
| 145 |
+
text = "\n".join(p.get("text", "") for p in pages).strip()
|
| 146 |
+
if text:
|
| 147 |
+
texts.append(f"[From: {f.name}]\n{text}")
|
| 148 |
+
total += len(text)
|
| 149 |
+
if total >= max_chars:
|
| 150 |
break
|
| 151 |
+
except Exception:
|
| 152 |
+
continue
|
| 153 |
+
|
| 154 |
+
combined = "\n\n".join(texts)
|
| 155 |
+
if len(combined) > max_chars:
|
| 156 |
+
combined = combined[:max_chars] + "..."
|
| 157 |
+
return combined
|
| 158 |
+
|
| 159 |
+
|
| 160 |
+
def _describe_columns_with_llm(
|
| 161 |
+
table_name: str,
|
| 162 |
+
source_file: str,
|
| 163 |
+
columns_info: list[dict],
|
| 164 |
+
codebook_text: str,
|
| 165 |
+
cfg: dict,
|
| 166 |
+
) -> dict:
|
| 167 |
+
"""Use LLM to generate column and table descriptions.
|
| 168 |
+
|
| 169 |
+
Args:
|
| 170 |
+
table_name: SQL table name.
|
| 171 |
+
source_file: Original file path relative to knowledge base.
|
| 172 |
+
columns_info: Column metadata dicts (name, type, sample, stats).
|
| 173 |
+
codebook_text: Text from codebook files (empty if none found).
|
| 174 |
+
cfg: App config for LLM access.
|
| 175 |
+
|
| 176 |
+
Returns:
|
| 177 |
+
Dict with 'table_description' and 'columns' mapping name -> description.
|
| 178 |
+
Empty dict on failure.
|
| 179 |
+
"""
|
| 180 |
+
from src.llm import generate
|
| 181 |
+
|
| 182 |
+
col_lines = []
|
| 183 |
+
for c in columns_info:
|
| 184 |
+
name = c["name"]
|
| 185 |
+
orig = c.get("original_name", name)
|
| 186 |
+
ctype = c.get("type", "TEXT")
|
| 187 |
+
samples = c.get("sample", [])
|
| 188 |
+
stats = c.get("stats", {})
|
| 189 |
+
|
| 190 |
+
parts = [f"- {name}"]
|
| 191 |
+
if orig != name:
|
| 192 |
+
parts.append(f"(originally: {orig})")
|
| 193 |
+
parts.append(f"[{ctype}]")
|
| 194 |
+
if stats.get("unique_count"):
|
| 195 |
+
parts.append(f"({stats['unique_count']} unique)")
|
| 196 |
+
if stats.get("min") is not None:
|
| 197 |
+
parts.append(f"range: {stats['min']}\u2013{stats['max']}")
|
| 198 |
+
if samples:
|
| 199 |
+
parts.append(f"e.g. {', '.join(str(s) for s in samples[:5])}")
|
| 200 |
+
|
| 201 |
+
col_lines.append(" ".join(parts))
|
| 202 |
+
|
| 203 |
+
cols_detail = "\n".join(col_lines)
|
| 204 |
+
|
| 205 |
+
if codebook_text:
|
| 206 |
+
codebook_section = (
|
| 207 |
+
"\nDocumentation/codebook found for this dataset:\n"
|
| 208 |
+
"--- CODEBOOK ---\n"
|
| 209 |
+
f"{codebook_text}\n"
|
| 210 |
+
"--- END CODEBOOK ---\n"
|
| 211 |
+
"Use the codebook to provide accurate column descriptions."
|
| 212 |
+
)
|
| 213 |
+
else:
|
| 214 |
+
codebook_section = (
|
| 215 |
+
"\nNo codebook found. Infer column meanings from names, "
|
| 216 |
+
"types, and sample values. Note what is inferred."
|
| 217 |
+
)
|
| 218 |
+
|
| 219 |
+
prompt = (
|
| 220 |
+
f"Analyze this dataset and describe what each column represents.\n\n"
|
| 221 |
+
f"Table: {table_name}\n"
|
| 222 |
+
f"Source: {source_file}\n\n"
|
| 223 |
+
f"Columns:\n{cols_detail}\n"
|
| 224 |
+
f"{codebook_section}\n\n"
|
| 225 |
+
"Return ONLY valid JSON (no markdown fences, no explanation):\n"
|
| 226 |
+
"{\n"
|
| 227 |
+
' "table_description": "What this table/dataset contains (1-2 sentences)",\n'
|
| 228 |
+
' "columns": {\n'
|
| 229 |
+
' "column_name": "What this column represents (1 sentence)"\n'
|
| 230 |
+
" }\n"
|
| 231 |
+
"}"
|
| 232 |
+
)
|
| 233 |
+
|
| 234 |
+
try:
|
| 235 |
+
response = generate(
|
| 236 |
+
"You are a data analyst. Describe dataset columns concisely and accurately.",
|
| 237 |
+
prompt,
|
| 238 |
+
cfg,
|
| 239 |
+
max_tokens=1024,
|
| 240 |
+
)
|
| 241 |
+
text = response.strip()
|
| 242 |
+
start = text.find("{")
|
| 243 |
+
end = text.rfind("}") + 1
|
| 244 |
+
if start >= 0 and end > start:
|
| 245 |
+
return json.loads(text[start:end])
|
| 246 |
+
return {}
|
| 247 |
+
except Exception:
|
| 248 |
+
return {}
|
| 249 |
+
|
| 250 |
+
|
| 251 |
+
def _enrich_schema_with_descriptions(
|
| 252 |
+
schema_registry: dict,
|
| 253 |
+
tabular_files: list[tuple],
|
| 254 |
+
documents_dir: str,
|
| 255 |
+
cfg: dict,
|
| 256 |
+
) -> None:
|
| 257 |
+
"""Add LLM-generated column descriptions to the schema registry.
|
| 258 |
+
|
| 259 |
+
For each table:
|
| 260 |
+
1. Search for codebook/documentation files near the source file.
|
| 261 |
+
2. If found, use LLM + codebook to describe columns.
|
| 262 |
+
3. If not, use LLM + column names/samples to infer descriptions.
|
| 263 |
+
|
| 264 |
+
Modifies schema_registry in place.
|
| 265 |
+
"""
|
| 266 |
+
# Build mapping from source_file (relative) -> absolute Path
|
| 267 |
+
source_to_path: dict[str, Path] = {}
|
| 268 |
+
for file_path, _dataset_name in tabular_files:
|
| 269 |
+
if file_path.suffix.lower() in SQL_EXTENSIONS:
|
| 270 |
+
try:
|
| 271 |
+
rel_path = str(file_path.relative_to(documents_dir))
|
| 272 |
+
source_to_path[rel_path] = file_path
|
| 273 |
+
except ValueError:
|
| 274 |
+
continue
|
| 275 |
+
|
| 276 |
+
for table_name, info in schema_registry.items():
|
| 277 |
+
source_file = info.get("source_file", "")
|
| 278 |
+
file_path = source_to_path.get(source_file)
|
| 279 |
+
|
| 280 |
+
# Find and read codebook files
|
| 281 |
+
codebook_text = ""
|
| 282 |
+
if file_path:
|
| 283 |
+
codebook_files = _find_codebook_files(file_path)
|
| 284 |
+
if codebook_files:
|
| 285 |
+
codebook_text = _read_codebook_text(codebook_files)
|
| 286 |
+
print(f" Found codebook for {table_name}: "
|
| 287 |
+
f"{[f.name for f in codebook_files]}")
|
| 288 |
+
|
| 289 |
+
# Get LLM descriptions
|
| 290 |
+
descriptions = _describe_columns_with_llm(
|
| 291 |
+
table_name, source_file, info["columns"], codebook_text, cfg,
|
| 292 |
+
)
|
| 293 |
+
|
| 294 |
+
if descriptions:
|
| 295 |
+
info["table_description"] = descriptions.get("table_description", "")
|
| 296 |
+
col_descs = descriptions.get("columns", {})
|
| 297 |
+
for col in info["columns"]:
|
| 298 |
+
col["description"] = col_descs.get(col["name"], "")
|
| 299 |
+
method = "codebook" if codebook_text else "inferred"
|
| 300 |
+
print(f" Described {table_name} columns ({method})")
|
| 301 |
+
else:
|
| 302 |
+
print(f" Could not generate descriptions for {table_name}")
|
| 303 |
|
| 304 |
|
| 305 |
def _load_rows_from_csv(file_path: str, ext: str) -> list[tuple]:
|
|
|
|
| 440 |
finally:
|
| 441 |
conn.close()
|
| 442 |
|
| 443 |
+
# Enrich with LLM-generated column descriptions
|
| 444 |
+
if schema_registry:
|
| 445 |
+
try:
|
| 446 |
+
_enrich_schema_with_descriptions(
|
| 447 |
+
schema_registry, tabular_files, documents_dir, cfg,
|
| 448 |
+
)
|
| 449 |
+
except Exception as e:
|
| 450 |
+
print(f" Column description generation failed (non-fatal): {e}")
|
| 451 |
+
|
| 452 |
with open(schema_path, "w", encoding="utf-8") as f:
|
| 453 |
json.dump(schema_registry, f, indent=2, default=str)
|
| 454 |
|
|
|
|
| 477 |
for sheet_or_name, headers, rows in tables:
|
| 478 |
table_name = _sanitize_table_name(dataset_name, file_path.stem, ext, sheet_or_name)
|
| 479 |
|
| 480 |
+
# Infer column types, collect samples and stats
|
| 481 |
safe_headers = [_sanitize_column_name(h) for h in headers]
|
| 482 |
col_types = []
|
| 483 |
col_samples = []
|
| 484 |
+
col_stats = []
|
| 485 |
for col_idx in range(len(headers)):
|
| 486 |
col_values = [row[col_idx] if col_idx < len(row) else None for row in rows]
|
| 487 |
+
ctype = _infer_column_type(col_values)
|
| 488 |
+
col_types.append(ctype)
|
| 489 |
col_samples.append(_get_sample_values(col_values))
|
| 490 |
+
col_stats.append(_get_column_stats(col_values, ctype))
|
| 491 |
|
| 492 |
# Create table
|
| 493 |
col_defs = ", ".join(f'"{h}" {t}' for h, t in zip(safe_headers, col_types))
|
|
|
|
| 530 |
"original_name": headers[i],
|
| 531 |
"type": col_types[i],
|
| 532 |
"sample": col_samples[i],
|
| 533 |
+
"stats": col_stats[i],
|
| 534 |
}
|
| 535 |
for i in range(len(headers))
|
| 536 |
],
|
src/sql_retriever.py
CHANGED
|
@@ -64,11 +64,18 @@ def execute_sql_query(sql_query: str, cfg: dict) -> list[dict]:
|
|
| 64 |
|
| 65 |
def format_sql_results_as_context(
|
| 66 |
rows: list[dict], sql_query: str, source_file: str,
|
|
|
|
| 67 |
) -> str:
|
| 68 |
"""Format SQL result rows as context for the verification pipeline.
|
| 69 |
|
| 70 |
Uses [CHUNK-SQL-NNN] IDs for internal anchoring, consistent with
|
| 71 |
CHUNK-LOCAL and CHUNK-WEB patterns.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 72 |
"""
|
| 73 |
if not rows:
|
| 74 |
return ""
|
|
@@ -79,6 +86,22 @@ def format_sql_results_as_context(
|
|
| 79 |
f"Source: {source_file}",
|
| 80 |
f"Rows returned: {len(rows)}\n",
|
| 81 |
]
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 82 |
for i, row in enumerate(rows, 1):
|
| 83 |
fields = ", ".join(f"{k} = {v}" for k, v in row.items())
|
| 84 |
parts.append(f"[CHUNK-SQL-{i:03d}] {fields}")
|
|
@@ -96,14 +119,9 @@ def _lookup_source_file(table_name: str, schema: dict) -> str:
|
|
| 96 |
def build_schema_summary(schema: dict) -> str:
|
| 97 |
"""Build a compact schema summary for injection into the QU prompt.
|
| 98 |
|
| 99 |
-
Includes
|
| 100 |
-
|
| 101 |
-
|
| 102 |
-
Format:
|
| 103 |
-
Available SQL tables:
|
| 104 |
-
- table_name (N rows, from source_file):
|
| 105 |
-
col1 (TYPE) e.g. "val1", "val2"
|
| 106 |
-
col2 (TYPE) e.g. "val1", "val2"
|
| 107 |
"""
|
| 108 |
if not schema:
|
| 109 |
return ""
|
|
@@ -112,12 +130,40 @@ def build_schema_summary(schema: dict) -> str:
|
|
| 112 |
for table_name, info in schema.items():
|
| 113 |
row_count = info.get("row_count", 0)
|
| 114 |
source = info.get("source_file", "")
|
| 115 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 116 |
for c in info.get("columns", []):
|
|
|
|
|
|
|
| 117 |
samples = c.get("sample", [])
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 118 |
sample_str = ""
|
| 119 |
if samples:
|
| 120 |
-
quoted = ", ".join(f'"{s}"' for s in samples[:
|
| 121 |
sample_str = f" e.g. {quoted}"
|
| 122 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 123 |
return "\n".join(lines)
|
|
|
|
| 64 |
|
| 65 |
def format_sql_results_as_context(
|
| 66 |
rows: list[dict], sql_query: str, source_file: str,
|
| 67 |
+
table_info: dict = None,
|
| 68 |
) -> str:
|
| 69 |
"""Format SQL result rows as context for the verification pipeline.
|
| 70 |
|
| 71 |
Uses [CHUNK-SQL-NNN] IDs for internal anchoring, consistent with
|
| 72 |
CHUNK-LOCAL and CHUNK-WEB patterns.
|
| 73 |
+
|
| 74 |
+
Args:
|
| 75 |
+
rows: Query result rows as dicts.
|
| 76 |
+
sql_query: The SQL query that produced these rows.
|
| 77 |
+
source_file: Original source file path.
|
| 78 |
+
table_info: Schema entry for the table (includes column descriptions).
|
| 79 |
"""
|
| 80 |
if not rows:
|
| 81 |
return ""
|
|
|
|
| 86 |
f"Source: {source_file}",
|
| 87 |
f"Rows returned: {len(rows)}\n",
|
| 88 |
]
|
| 89 |
+
|
| 90 |
+
# Include table and column descriptions so the LLM understands the data
|
| 91 |
+
if table_info:
|
| 92 |
+
table_desc = table_info.get("table_description", "")
|
| 93 |
+
if table_desc:
|
| 94 |
+
parts.append(f"Dataset description: {table_desc}\n")
|
| 95 |
+
col_descs = {
|
| 96 |
+
c["name"]: c.get("description", "")
|
| 97 |
+
for c in table_info.get("columns", [])
|
| 98 |
+
}
|
| 99 |
+
desc_lines = [f" {name}: {desc}" for name, desc in col_descs.items() if desc]
|
| 100 |
+
if desc_lines:
|
| 101 |
+
parts.append("Column descriptions:")
|
| 102 |
+
parts.extend(desc_lines)
|
| 103 |
+
parts.append("")
|
| 104 |
+
|
| 105 |
for i, row in enumerate(rows, 1):
|
| 106 |
fields = ", ".join(f"{k} = {v}" for k, v in row.items())
|
| 107 |
parts.append(f"[CHUNK-SQL-{i:03d}] {fields}")
|
|
|
|
| 119 |
def build_schema_summary(schema: dict) -> str:
|
| 120 |
"""Build a compact schema summary for injection into the QU prompt.
|
| 121 |
|
| 122 |
+
Includes column descriptions (from codebook or LLM inference),
|
| 123 |
+
sample values, and statistics so the LLM knows what data looks like
|
| 124 |
+
for correct WHERE clauses and aggregation queries.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 125 |
"""
|
| 126 |
if not schema:
|
| 127 |
return ""
|
|
|
|
| 130 |
for table_name, info in schema.items():
|
| 131 |
row_count = info.get("row_count", 0)
|
| 132 |
source = info.get("source_file", "")
|
| 133 |
+
table_desc = info.get("table_description", "")
|
| 134 |
+
|
| 135 |
+
header = f"\n- {table_name} ({row_count} rows, from {source})"
|
| 136 |
+
if table_desc:
|
| 137 |
+
header += f"\n Description: {table_desc}"
|
| 138 |
+
lines.append(header + ":")
|
| 139 |
+
|
| 140 |
for c in info.get("columns", []):
|
| 141 |
+
col_type = c.get("type", "TEXT")
|
| 142 |
+
stats = c.get("stats", {})
|
| 143 |
samples = c.get("sample", [])
|
| 144 |
+
desc = c.get("description", "")
|
| 145 |
+
|
| 146 |
+
# Build metadata string
|
| 147 |
+
meta_parts = [col_type]
|
| 148 |
+
unique_count = stats.get("unique_count")
|
| 149 |
+
col_min = stats.get("min")
|
| 150 |
+
col_max = stats.get("max")
|
| 151 |
+
|
| 152 |
+
if col_min is not None and col_max is not None:
|
| 153 |
+
meta_parts.append(f"range {col_min}\u2013{col_max}")
|
| 154 |
+
if unique_count is not None:
|
| 155 |
+
meta_parts.append(f"{unique_count} unique")
|
| 156 |
+
|
| 157 |
+
meta_str = ", ".join(meta_parts)
|
| 158 |
+
|
| 159 |
+
# Build sample string
|
| 160 |
sample_str = ""
|
| 161 |
if samples:
|
| 162 |
+
quoted = ", ".join(f'"{s}"' for s in samples[:5])
|
| 163 |
sample_str = f" e.g. {quoted}"
|
| 164 |
+
|
| 165 |
+
line = f" {c['name']} ({meta_str}){sample_str}"
|
| 166 |
+
if desc:
|
| 167 |
+
line += f" \u2014 {desc}"
|
| 168 |
+
lines.append(line)
|
| 169 |
return "\n".join(lines)
|