LIANJie-Jason Claude Opus 4.6 commited on
Commit
3d1d5b4
·
1 Parent(s): 353f65a

feat: add LLM-powered column descriptions with codebook detection

Browse files

During SQL ingestion, each table columns are now described by the LLM:
1. Search same directory for codebook/documentation files (PDF, DOCX, TXT, MD)
2. If codebook found, pass its text + column metadata to LLM for descriptions
3. If no codebook, LLM infers column meanings from names, types, samples
4. Descriptions stored in sql_schemas.json alongside existing column metadata

Column descriptions are always visible to the chatbot:
- build_schema_summary() includes descriptions in the QU prompt
- format_sql_results_as_context() includes descriptions with SQL results
- Graceful degradation: LLM failure is non-fatal

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

Files changed (3) hide show
  1. src/retriever.py +4 -1
  2. src/sql_ingest.py +257 -8
  3. src/sql_retriever.py +57 -11
src/retriever.py CHANGED
@@ -171,7 +171,10 @@ def _run_sql_retrieval(sql_query: str, cfg: dict) -> tuple[list[dict], str]:
171
  pass
172
  table_name = _extract_table_from_query(sql_query)
173
  source_file = _lookup_source_file(table_name, schema)
174
- sql_context = format_sql_results_as_context(sql_rows, sql_query, source_file)
 
 
 
175
  return sql_rows, sql_context
176
 
177
 
 
171
  pass
172
  table_name = _extract_table_from_query(sql_query)
173
  source_file = _lookup_source_file(table_name, schema)
174
+ table_info = schema.get(table_name)
175
+ sql_context = format_sql_results_as_context(
176
+ sql_rows, sql_query, source_file, table_info=table_info,
177
+ )
178
  return sql_rows, sql_context
179
 
180
 
src/sql_ingest.py CHANGED
@@ -10,6 +10,15 @@ from pathlib import Path
10
  # Extensions that trigger SQL ingestion (tabular formats)
11
  SQL_EXTENSIONS = {".csv", ".tab", ".tsv", ".xlsx", ".xls", ".dta", ".sav", ".rds", ".rda"}
12
 
 
 
 
 
 
 
 
 
 
13
 
14
  def _sanitize_part(s: str) -> str:
15
  """Sanitize a single name part: replace non-alnum with _, collapse, strip."""
@@ -51,19 +60,246 @@ def _infer_column_type(values: list) -> str:
51
  return "TEXT"
52
 
53
 
54
- def _get_sample_values(values: list, n: int = 3) -> list:
55
- """Get up to n unique non-empty sample values."""
 
 
 
 
 
 
56
  seen = set()
57
- samples = []
58
  for v in values:
59
  if v is not None and str(v).strip() and str(v).lower() != "nan":
60
  v_str = str(v).strip()
61
  if v_str not in seen:
62
  seen.add(v_str)
63
- samples.append(v_str)
64
- if len(samples) >= n:
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
65
  break
66
- return samples
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
67
 
68
 
69
  def _load_rows_from_csv(file_path: str, ext: str) -> list[tuple]:
@@ -204,6 +440,15 @@ def ingest_to_sql(files: list[tuple], documents_dir: str, cfg: dict) -> dict:
204
  finally:
205
  conn.close()
206
 
 
 
 
 
 
 
 
 
 
207
  with open(schema_path, "w", encoding="utf-8") as f:
208
  json.dump(schema_registry, f, indent=2, default=str)
209
 
@@ -232,14 +477,17 @@ def _ingest_tables(
232
  for sheet_or_name, headers, rows in tables:
233
  table_name = _sanitize_table_name(dataset_name, file_path.stem, ext, sheet_or_name)
234
 
235
- # Infer column types and collect samples
236
  safe_headers = [_sanitize_column_name(h) for h in headers]
237
  col_types = []
238
  col_samples = []
 
239
  for col_idx in range(len(headers)):
240
  col_values = [row[col_idx] if col_idx < len(row) else None for row in rows]
241
- col_types.append(_infer_column_type(col_values))
 
242
  col_samples.append(_get_sample_values(col_values))
 
243
 
244
  # Create table
245
  col_defs = ", ".join(f'"{h}" {t}' for h, t in zip(safe_headers, col_types))
@@ -282,6 +530,7 @@ def _ingest_tables(
282
  "original_name": headers[i],
283
  "type": col_types[i],
284
  "sample": col_samples[i],
 
285
  }
286
  for i in range(len(headers))
287
  ],
 
10
  # Extensions that trigger SQL ingestion (tabular formats)
11
  SQL_EXTENSIONS = {".csv", ".tab", ".tsv", ".xlsx", ".xls", ".dta", ".sav", ".rds", ".rda"}
12
 
13
+ # Extensions that might contain codebook/documentation
14
+ _CODEBOOK_EXTENSIONS = {".pdf", ".docx", ".txt", ".md"}
15
+
16
+ # Keywords suggesting a file is a codebook
17
+ _CODEBOOK_KEYWORDS = {
18
+ "codebook", "readme", "dictionary", "documentation",
19
+ "manual", "guide", "metadata", "variable", "description",
20
+ }
21
+
22
 
23
  def _sanitize_part(s: str) -> str:
24
  """Sanitize a single name part: replace non-alnum with _, collapse, strip."""
 
60
  return "TEXT"
61
 
62
 
63
+ def _get_sample_values(values: list, n: int = 5) -> list:
64
+ """Get a representative spread of up to n unique non-empty sample values.
65
+
66
+ Instead of taking the first n values (which may be biased by sort order),
67
+ picks evenly spaced values from the sorted unique set so the LLM sees
68
+ the full range (e.g., first, middle, last country names).
69
+ """
70
+ unique = []
71
  seen = set()
 
72
  for v in values:
73
  if v is not None and str(v).strip() and str(v).lower() != "nan":
74
  v_str = str(v).strip()
75
  if v_str not in seen:
76
  seen.add(v_str)
77
+ unique.append(v_str)
78
+ if not unique:
79
+ return []
80
+ sorted_vals = sorted(unique)
81
+ if len(sorted_vals) <= n:
82
+ return sorted_vals
83
+ # Pick evenly spaced indices including first and last
84
+ indices = [round(i * (len(sorted_vals) - 1) / (n - 1)) for i in range(n)]
85
+ return [sorted_vals[i] for i in indices]
86
+
87
+
88
+ def _get_column_stats(values: list, col_type: str) -> dict:
89
+ """Compute column statistics for schema metadata.
90
+
91
+ Returns dict with 'unique_count' and optionally 'min'/'max' for numerics.
92
+ """
93
+ non_empty = [
94
+ str(v).strip() for v in values
95
+ if v is not None and str(v).strip() and str(v).lower() != "nan"
96
+ ]
97
+ unique_count = len(set(non_empty))
98
+ stats = {"unique_count": unique_count}
99
+ if col_type == "INTEGER" and non_empty:
100
+ try:
101
+ nums = [int(v) for v in non_empty]
102
+ stats["min"] = min(nums)
103
+ stats["max"] = max(nums)
104
+ except (ValueError, TypeError):
105
+ pass
106
+ elif col_type == "REAL" and non_empty:
107
+ try:
108
+ nums = [float(v) for v in non_empty]
109
+ stats["min"] = min(nums)
110
+ stats["max"] = max(nums)
111
+ except (ValueError, TypeError):
112
+ pass
113
+ return stats
114
+
115
+
116
+ def _find_codebook_files(tabular_file: Path) -> list[Path]:
117
+ """Search for codebook/documentation files in the same directory as the tabular file."""
118
+ parent = tabular_file.parent
119
+ candidates = []
120
+ for f in sorted(parent.iterdir()):
121
+ if not f.is_file():
122
+ continue
123
+ if f.suffix.lower() not in _CODEBOOK_EXTENSIONS:
124
+ continue
125
+ candidates.append(f)
126
+
127
+ # Sort: files with codebook keywords in name come first
128
+ def priority(p):
129
+ name_lower = p.stem.lower()
130
+ return 0 if any(kw in name_lower for kw in _CODEBOOK_KEYWORDS) else 1
131
+
132
+ candidates.sort(key=priority)
133
+ return candidates
134
+
135
+
136
+ def _read_codebook_text(codebook_files: list[Path], max_chars: int = 4000) -> str:
137
+ """Read codebook files and return concatenated text, truncated."""
138
+ from src.readers import read_file
139
+
140
+ texts = []
141
+ total = 0
142
+ for f in codebook_files:
143
+ try:
144
+ pages = read_file(str(f))
145
+ text = "\n".join(p.get("text", "") for p in pages).strip()
146
+ if text:
147
+ texts.append(f"[From: {f.name}]\n{text}")
148
+ total += len(text)
149
+ if total >= max_chars:
150
  break
151
+ except Exception:
152
+ continue
153
+
154
+ combined = "\n\n".join(texts)
155
+ if len(combined) > max_chars:
156
+ combined = combined[:max_chars] + "..."
157
+ return combined
158
+
159
+
160
+ def _describe_columns_with_llm(
161
+ table_name: str,
162
+ source_file: str,
163
+ columns_info: list[dict],
164
+ codebook_text: str,
165
+ cfg: dict,
166
+ ) -> dict:
167
+ """Use LLM to generate column and table descriptions.
168
+
169
+ Args:
170
+ table_name: SQL table name.
171
+ source_file: Original file path relative to knowledge base.
172
+ columns_info: Column metadata dicts (name, type, sample, stats).
173
+ codebook_text: Text from codebook files (empty if none found).
174
+ cfg: App config for LLM access.
175
+
176
+ Returns:
177
+ Dict with 'table_description' and 'columns' mapping name -> description.
178
+ Empty dict on failure.
179
+ """
180
+ from src.llm import generate
181
+
182
+ col_lines = []
183
+ for c in columns_info:
184
+ name = c["name"]
185
+ orig = c.get("original_name", name)
186
+ ctype = c.get("type", "TEXT")
187
+ samples = c.get("sample", [])
188
+ stats = c.get("stats", {})
189
+
190
+ parts = [f"- {name}"]
191
+ if orig != name:
192
+ parts.append(f"(originally: {orig})")
193
+ parts.append(f"[{ctype}]")
194
+ if stats.get("unique_count"):
195
+ parts.append(f"({stats['unique_count']} unique)")
196
+ if stats.get("min") is not None:
197
+ parts.append(f"range: {stats['min']}\u2013{stats['max']}")
198
+ if samples:
199
+ parts.append(f"e.g. {', '.join(str(s) for s in samples[:5])}")
200
+
201
+ col_lines.append(" ".join(parts))
202
+
203
+ cols_detail = "\n".join(col_lines)
204
+
205
+ if codebook_text:
206
+ codebook_section = (
207
+ "\nDocumentation/codebook found for this dataset:\n"
208
+ "--- CODEBOOK ---\n"
209
+ f"{codebook_text}\n"
210
+ "--- END CODEBOOK ---\n"
211
+ "Use the codebook to provide accurate column descriptions."
212
+ )
213
+ else:
214
+ codebook_section = (
215
+ "\nNo codebook found. Infer column meanings from names, "
216
+ "types, and sample values. Note what is inferred."
217
+ )
218
+
219
+ prompt = (
220
+ f"Analyze this dataset and describe what each column represents.\n\n"
221
+ f"Table: {table_name}\n"
222
+ f"Source: {source_file}\n\n"
223
+ f"Columns:\n{cols_detail}\n"
224
+ f"{codebook_section}\n\n"
225
+ "Return ONLY valid JSON (no markdown fences, no explanation):\n"
226
+ "{\n"
227
+ ' "table_description": "What this table/dataset contains (1-2 sentences)",\n'
228
+ ' "columns": {\n'
229
+ ' "column_name": "What this column represents (1 sentence)"\n'
230
+ " }\n"
231
+ "}"
232
+ )
233
+
234
+ try:
235
+ response = generate(
236
+ "You are a data analyst. Describe dataset columns concisely and accurately.",
237
+ prompt,
238
+ cfg,
239
+ max_tokens=1024,
240
+ )
241
+ text = response.strip()
242
+ start = text.find("{")
243
+ end = text.rfind("}") + 1
244
+ if start >= 0 and end > start:
245
+ return json.loads(text[start:end])
246
+ return {}
247
+ except Exception:
248
+ return {}
249
+
250
+
251
+ def _enrich_schema_with_descriptions(
252
+ schema_registry: dict,
253
+ tabular_files: list[tuple],
254
+ documents_dir: str,
255
+ cfg: dict,
256
+ ) -> None:
257
+ """Add LLM-generated column descriptions to the schema registry.
258
+
259
+ For each table:
260
+ 1. Search for codebook/documentation files near the source file.
261
+ 2. If found, use LLM + codebook to describe columns.
262
+ 3. If not, use LLM + column names/samples to infer descriptions.
263
+
264
+ Modifies schema_registry in place.
265
+ """
266
+ # Build mapping from source_file (relative) -> absolute Path
267
+ source_to_path: dict[str, Path] = {}
268
+ for file_path, _dataset_name in tabular_files:
269
+ if file_path.suffix.lower() in SQL_EXTENSIONS:
270
+ try:
271
+ rel_path = str(file_path.relative_to(documents_dir))
272
+ source_to_path[rel_path] = file_path
273
+ except ValueError:
274
+ continue
275
+
276
+ for table_name, info in schema_registry.items():
277
+ source_file = info.get("source_file", "")
278
+ file_path = source_to_path.get(source_file)
279
+
280
+ # Find and read codebook files
281
+ codebook_text = ""
282
+ if file_path:
283
+ codebook_files = _find_codebook_files(file_path)
284
+ if codebook_files:
285
+ codebook_text = _read_codebook_text(codebook_files)
286
+ print(f" Found codebook for {table_name}: "
287
+ f"{[f.name for f in codebook_files]}")
288
+
289
+ # Get LLM descriptions
290
+ descriptions = _describe_columns_with_llm(
291
+ table_name, source_file, info["columns"], codebook_text, cfg,
292
+ )
293
+
294
+ if descriptions:
295
+ info["table_description"] = descriptions.get("table_description", "")
296
+ col_descs = descriptions.get("columns", {})
297
+ for col in info["columns"]:
298
+ col["description"] = col_descs.get(col["name"], "")
299
+ method = "codebook" if codebook_text else "inferred"
300
+ print(f" Described {table_name} columns ({method})")
301
+ else:
302
+ print(f" Could not generate descriptions for {table_name}")
303
 
304
 
305
  def _load_rows_from_csv(file_path: str, ext: str) -> list[tuple]:
 
440
  finally:
441
  conn.close()
442
 
443
+ # Enrich with LLM-generated column descriptions
444
+ if schema_registry:
445
+ try:
446
+ _enrich_schema_with_descriptions(
447
+ schema_registry, tabular_files, documents_dir, cfg,
448
+ )
449
+ except Exception as e:
450
+ print(f" Column description generation failed (non-fatal): {e}")
451
+
452
  with open(schema_path, "w", encoding="utf-8") as f:
453
  json.dump(schema_registry, f, indent=2, default=str)
454
 
 
477
  for sheet_or_name, headers, rows in tables:
478
  table_name = _sanitize_table_name(dataset_name, file_path.stem, ext, sheet_or_name)
479
 
480
+ # Infer column types, collect samples and stats
481
  safe_headers = [_sanitize_column_name(h) for h in headers]
482
  col_types = []
483
  col_samples = []
484
+ col_stats = []
485
  for col_idx in range(len(headers)):
486
  col_values = [row[col_idx] if col_idx < len(row) else None for row in rows]
487
+ ctype = _infer_column_type(col_values)
488
+ col_types.append(ctype)
489
  col_samples.append(_get_sample_values(col_values))
490
+ col_stats.append(_get_column_stats(col_values, ctype))
491
 
492
  # Create table
493
  col_defs = ", ".join(f'"{h}" {t}' for h, t in zip(safe_headers, col_types))
 
530
  "original_name": headers[i],
531
  "type": col_types[i],
532
  "sample": col_samples[i],
533
+ "stats": col_stats[i],
534
  }
535
  for i in range(len(headers))
536
  ],
src/sql_retriever.py CHANGED
@@ -64,11 +64,18 @@ def execute_sql_query(sql_query: str, cfg: dict) -> list[dict]:
64
 
65
  def format_sql_results_as_context(
66
  rows: list[dict], sql_query: str, source_file: str,
 
67
  ) -> str:
68
  """Format SQL result rows as context for the verification pipeline.
69
 
70
  Uses [CHUNK-SQL-NNN] IDs for internal anchoring, consistent with
71
  CHUNK-LOCAL and CHUNK-WEB patterns.
 
 
 
 
 
 
72
  """
73
  if not rows:
74
  return ""
@@ -79,6 +86,22 @@ def format_sql_results_as_context(
79
  f"Source: {source_file}",
80
  f"Rows returned: {len(rows)}\n",
81
  ]
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
82
  for i, row in enumerate(rows, 1):
83
  fields = ", ".join(f"{k} = {v}" for k, v in row.items())
84
  parts.append(f"[CHUNK-SQL-{i:03d}] {fields}")
@@ -96,14 +119,9 @@ def _lookup_source_file(table_name: str, schema: dict) -> str:
96
  def build_schema_summary(schema: dict) -> str:
97
  """Build a compact schema summary for injection into the QU prompt.
98
 
99
- Includes sample values so the LLM knows what data looks like
100
- (e.g., country names, value formats) for correct WHERE clauses.
101
-
102
- Format:
103
- Available SQL tables:
104
- - table_name (N rows, from source_file):
105
- col1 (TYPE) e.g. "val1", "val2"
106
- col2 (TYPE) e.g. "val1", "val2"
107
  """
108
  if not schema:
109
  return ""
@@ -112,12 +130,40 @@ def build_schema_summary(schema: dict) -> str:
112
  for table_name, info in schema.items():
113
  row_count = info.get("row_count", 0)
114
  source = info.get("source_file", "")
115
- lines.append(f"\n- {table_name} ({row_count} rows, from {source}):")
 
 
 
 
 
 
116
  for c in info.get("columns", []):
 
 
117
  samples = c.get("sample", [])
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
118
  sample_str = ""
119
  if samples:
120
- quoted = ", ".join(f'"{s}"' for s in samples[:3])
121
  sample_str = f" e.g. {quoted}"
122
- lines.append(f" {c['name']} ({c['type']}){sample_str}")
 
 
 
 
123
  return "\n".join(lines)
 
64
 
65
  def format_sql_results_as_context(
66
  rows: list[dict], sql_query: str, source_file: str,
67
+ table_info: dict = None,
68
  ) -> str:
69
  """Format SQL result rows as context for the verification pipeline.
70
 
71
  Uses [CHUNK-SQL-NNN] IDs for internal anchoring, consistent with
72
  CHUNK-LOCAL and CHUNK-WEB patterns.
73
+
74
+ Args:
75
+ rows: Query result rows as dicts.
76
+ sql_query: The SQL query that produced these rows.
77
+ source_file: Original source file path.
78
+ table_info: Schema entry for the table (includes column descriptions).
79
  """
80
  if not rows:
81
  return ""
 
86
  f"Source: {source_file}",
87
  f"Rows returned: {len(rows)}\n",
88
  ]
89
+
90
+ # Include table and column descriptions so the LLM understands the data
91
+ if table_info:
92
+ table_desc = table_info.get("table_description", "")
93
+ if table_desc:
94
+ parts.append(f"Dataset description: {table_desc}\n")
95
+ col_descs = {
96
+ c["name"]: c.get("description", "")
97
+ for c in table_info.get("columns", [])
98
+ }
99
+ desc_lines = [f" {name}: {desc}" for name, desc in col_descs.items() if desc]
100
+ if desc_lines:
101
+ parts.append("Column descriptions:")
102
+ parts.extend(desc_lines)
103
+ parts.append("")
104
+
105
  for i, row in enumerate(rows, 1):
106
  fields = ", ".join(f"{k} = {v}" for k, v in row.items())
107
  parts.append(f"[CHUNK-SQL-{i:03d}] {fields}")
 
119
  def build_schema_summary(schema: dict) -> str:
120
  """Build a compact schema summary for injection into the QU prompt.
121
 
122
+ Includes column descriptions (from codebook or LLM inference),
123
+ sample values, and statistics so the LLM knows what data looks like
124
+ for correct WHERE clauses and aggregation queries.
 
 
 
 
 
125
  """
126
  if not schema:
127
  return ""
 
130
  for table_name, info in schema.items():
131
  row_count = info.get("row_count", 0)
132
  source = info.get("source_file", "")
133
+ table_desc = info.get("table_description", "")
134
+
135
+ header = f"\n- {table_name} ({row_count} rows, from {source})"
136
+ if table_desc:
137
+ header += f"\n Description: {table_desc}"
138
+ lines.append(header + ":")
139
+
140
  for c in info.get("columns", []):
141
+ col_type = c.get("type", "TEXT")
142
+ stats = c.get("stats", {})
143
  samples = c.get("sample", [])
144
+ desc = c.get("description", "")
145
+
146
+ # Build metadata string
147
+ meta_parts = [col_type]
148
+ unique_count = stats.get("unique_count")
149
+ col_min = stats.get("min")
150
+ col_max = stats.get("max")
151
+
152
+ if col_min is not None and col_max is not None:
153
+ meta_parts.append(f"range {col_min}\u2013{col_max}")
154
+ if unique_count is not None:
155
+ meta_parts.append(f"{unique_count} unique")
156
+
157
+ meta_str = ", ".join(meta_parts)
158
+
159
+ # Build sample string
160
  sample_str = ""
161
  if samples:
162
+ quoted = ", ".join(f'"{s}"' for s in samples[:5])
163
  sample_str = f" e.g. {quoted}"
164
+
165
+ line = f" {c['name']} ({meta_str}){sample_str}"
166
+ if desc:
167
+ line += f" \u2014 {desc}"
168
+ lines.append(line)
169
  return "\n".join(lines)