LIANJie-Jason Claude Opus 4.6 commited on
Commit
b71f629
·
1 Parent(s): 5ccab87

fix: try alternate TEXT columns before word-level fuzzy matching

Browse files

When phrase-level fuzzy fails on the original column (e.g. Country),
now tries phrase-level LIKE on other TEXT columns in the same table
before falling back to word-level fuzzy. This fixes South Korea queries
returning 98 rows (both Koreas via LIKE '%Korea%') instead of 49 rows
(just South Korea via Country_OLD LIKE '%South Korea%').

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

Files changed (1) hide show
  1. src/retriever.py +79 -0
src/retriever.py CHANGED
@@ -202,6 +202,78 @@ def _build_fallback_sql_query(query: str, cfg: dict) -> str | None:
202
  return None
203
 
204
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
205
  def _run_sql_retrieval(sql_query: str, cfg: dict) -> tuple[list[dict], str, str]:
206
  """Execute SQL query and format results as context.
207
 
@@ -234,6 +306,13 @@ def _run_sql_retrieval(sql_query: str, cfg: dict) -> tuple[list[dict], str, str]
234
  effective_query = fuzzy
235
  match_type = "fuzzy (phrase)"
236
 
 
 
 
 
 
 
 
237
  # Step 3: if still no results, try word-level fuzzy (LIKE '%Korea%')
238
  if not sql_rows:
239
  word_fuzzy = make_fuzzy_query(sql_query, word_level=True)
 
202
  return None
203
 
204
 
205
+ def _try_alternate_columns(sql_query: str, cfg: dict) -> tuple[list[dict], str] | None:
206
+ """Try phrase-level LIKE on other TEXT columns of the same table.
207
+
208
+ When ``WHERE "Country" LIKE '%South Korea%'`` fails because the value
209
+ is stored in a different column (e.g. ``Country_OLD``), this function
210
+ tries each TEXT column in the table until one matches.
211
+
212
+ Returns (rows, effective_query) or None if nothing found.
213
+ """
214
+ import json as _json
215
+ import os
216
+
217
+ # Extract table name and search value from the query
218
+ table_name = _extract_table_from_query(sql_query)
219
+ if not table_name:
220
+ return None
221
+
222
+ # Extract the LIKE value or = value from the WHERE clause
223
+ value_match = re.search(
224
+ r"LIKE\s+'%([^%]+)%'|=\s*'([^']+)'",
225
+ sql_query, re.IGNORECASE,
226
+ )
227
+ if not value_match:
228
+ return None
229
+ search_value = value_match.group(1) or value_match.group(2)
230
+ if not search_value:
231
+ return None
232
+
233
+ # Extract the column currently being searched
234
+ col_match = re.search(
235
+ r'["`]?(\w+)["`]?\s*(?:LIKE|=)\s*',
236
+ sql_query, re.IGNORECASE,
237
+ )
238
+ current_col = col_match.group(1) if col_match else ""
239
+
240
+ # Load schema to find other TEXT columns
241
+ sql_db_dir = cfg.get("paths", {}).get("sql_db", "sql_db")
242
+ if not os.path.isabs(sql_db_dir):
243
+ from pathlib import Path
244
+ project_root = Path(__file__).resolve().parent.parent
245
+ sql_db_dir = os.path.join(str(project_root), sql_db_dir)
246
+ schema_path = os.path.join(sql_db_dir, "sql_schemas.json")
247
+ try:
248
+ with open(schema_path) as f:
249
+ schema = _json.load(f)
250
+ except Exception:
251
+ return None
252
+
253
+ table_info = schema.get(table_name)
254
+ if not table_info:
255
+ return None
256
+
257
+ text_cols = [
258
+ c.get("name") for c in table_info.get("columns", [])
259
+ if c.get("type", "").upper() == "TEXT"
260
+ and c.get("name")
261
+ and c.get("name") != current_col
262
+ ]
263
+
264
+ from src.sql_retriever import execute_sql_query
265
+
266
+ max_rows = cfg.get("sql", {}).get("max_rows", 200)
267
+ safe_value = search_value.replace("'", "''")
268
+ for col in text_cols:
269
+ alt_query = f'SELECT * FROM "{table_name}" WHERE "{col}" LIKE \'%{safe_value}%\' LIMIT {max_rows}'
270
+ rows = execute_sql_query(alt_query, cfg)
271
+ if rows:
272
+ return rows, alt_query
273
+
274
+ return None
275
+
276
+
277
  def _run_sql_retrieval(sql_query: str, cfg: dict) -> tuple[list[dict], str, str]:
278
  """Execute SQL query and format results as context.
279
 
 
306
  effective_query = fuzzy
307
  match_type = "fuzzy (phrase)"
308
 
309
+ # Step 2.5: if phrase fuzzy failed on original column, try other TEXT columns
310
+ if not sql_rows:
311
+ alt = _try_alternate_columns(fuzzy or sql_query, cfg)
312
+ if alt:
313
+ sql_rows, effective_query = alt
314
+ match_type = "fuzzy (alt column)"
315
+
316
  # Step 3: if still no results, try word-level fuzzy (LIKE '%Korea%')
317
  if not sql_rows:
318
  word_fuzzy = make_fuzzy_query(sql_query, word_level=True)