Spaces:
Paused
Paused
Commit ·
b71f629
1
Parent(s): 5ccab87
fix: try alternate TEXT columns before word-level fuzzy matching
Browse filesWhen phrase-level fuzzy fails on the original column (e.g. Country),
now tries phrase-level LIKE on other TEXT columns in the same table
before falling back to word-level fuzzy. This fixes South Korea queries
returning 98 rows (both Koreas via LIKE '%Korea%') instead of 49 rows
(just South Korea via Country_OLD LIKE '%South Korea%').
Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
- src/retriever.py +79 -0
src/retriever.py
CHANGED
|
@@ -202,6 +202,78 @@ def _build_fallback_sql_query(query: str, cfg: dict) -> str | None:
|
|
| 202 |
return None
|
| 203 |
|
| 204 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 205 |
def _run_sql_retrieval(sql_query: str, cfg: dict) -> tuple[list[dict], str, str]:
|
| 206 |
"""Execute SQL query and format results as context.
|
| 207 |
|
|
@@ -234,6 +306,13 @@ def _run_sql_retrieval(sql_query: str, cfg: dict) -> tuple[list[dict], str, str]
|
|
| 234 |
effective_query = fuzzy
|
| 235 |
match_type = "fuzzy (phrase)"
|
| 236 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 237 |
# Step 3: if still no results, try word-level fuzzy (LIKE '%Korea%')
|
| 238 |
if not sql_rows:
|
| 239 |
word_fuzzy = make_fuzzy_query(sql_query, word_level=True)
|
|
|
|
| 202 |
return None
|
| 203 |
|
| 204 |
|
| 205 |
+
def _try_alternate_columns(sql_query: str, cfg: dict) -> tuple[list[dict], str] | None:
|
| 206 |
+
"""Try phrase-level LIKE on other TEXT columns of the same table.
|
| 207 |
+
|
| 208 |
+
When ``WHERE "Country" LIKE '%South Korea%'`` fails because the value
|
| 209 |
+
is stored in a different column (e.g. ``Country_OLD``), this function
|
| 210 |
+
tries each TEXT column in the table until one matches.
|
| 211 |
+
|
| 212 |
+
Returns (rows, effective_query) or None if nothing found.
|
| 213 |
+
"""
|
| 214 |
+
import json as _json
|
| 215 |
+
import os
|
| 216 |
+
|
| 217 |
+
# Extract table name and search value from the query
|
| 218 |
+
table_name = _extract_table_from_query(sql_query)
|
| 219 |
+
if not table_name:
|
| 220 |
+
return None
|
| 221 |
+
|
| 222 |
+
# Extract the LIKE value or = value from the WHERE clause
|
| 223 |
+
value_match = re.search(
|
| 224 |
+
r"LIKE\s+'%([^%]+)%'|=\s*'([^']+)'",
|
| 225 |
+
sql_query, re.IGNORECASE,
|
| 226 |
+
)
|
| 227 |
+
if not value_match:
|
| 228 |
+
return None
|
| 229 |
+
search_value = value_match.group(1) or value_match.group(2)
|
| 230 |
+
if not search_value:
|
| 231 |
+
return None
|
| 232 |
+
|
| 233 |
+
# Extract the column currently being searched
|
| 234 |
+
col_match = re.search(
|
| 235 |
+
r'["`]?(\w+)["`]?\s*(?:LIKE|=)\s*',
|
| 236 |
+
sql_query, re.IGNORECASE,
|
| 237 |
+
)
|
| 238 |
+
current_col = col_match.group(1) if col_match else ""
|
| 239 |
+
|
| 240 |
+
# Load schema to find other TEXT columns
|
| 241 |
+
sql_db_dir = cfg.get("paths", {}).get("sql_db", "sql_db")
|
| 242 |
+
if not os.path.isabs(sql_db_dir):
|
| 243 |
+
from pathlib import Path
|
| 244 |
+
project_root = Path(__file__).resolve().parent.parent
|
| 245 |
+
sql_db_dir = os.path.join(str(project_root), sql_db_dir)
|
| 246 |
+
schema_path = os.path.join(sql_db_dir, "sql_schemas.json")
|
| 247 |
+
try:
|
| 248 |
+
with open(schema_path) as f:
|
| 249 |
+
schema = _json.load(f)
|
| 250 |
+
except Exception:
|
| 251 |
+
return None
|
| 252 |
+
|
| 253 |
+
table_info = schema.get(table_name)
|
| 254 |
+
if not table_info:
|
| 255 |
+
return None
|
| 256 |
+
|
| 257 |
+
text_cols = [
|
| 258 |
+
c.get("name") for c in table_info.get("columns", [])
|
| 259 |
+
if c.get("type", "").upper() == "TEXT"
|
| 260 |
+
and c.get("name")
|
| 261 |
+
and c.get("name") != current_col
|
| 262 |
+
]
|
| 263 |
+
|
| 264 |
+
from src.sql_retriever import execute_sql_query
|
| 265 |
+
|
| 266 |
+
max_rows = cfg.get("sql", {}).get("max_rows", 200)
|
| 267 |
+
safe_value = search_value.replace("'", "''")
|
| 268 |
+
for col in text_cols:
|
| 269 |
+
alt_query = f'SELECT * FROM "{table_name}" WHERE "{col}" LIKE \'%{safe_value}%\' LIMIT {max_rows}'
|
| 270 |
+
rows = execute_sql_query(alt_query, cfg)
|
| 271 |
+
if rows:
|
| 272 |
+
return rows, alt_query
|
| 273 |
+
|
| 274 |
+
return None
|
| 275 |
+
|
| 276 |
+
|
| 277 |
def _run_sql_retrieval(sql_query: str, cfg: dict) -> tuple[list[dict], str, str]:
|
| 278 |
"""Execute SQL query and format results as context.
|
| 279 |
|
|
|
|
| 306 |
effective_query = fuzzy
|
| 307 |
match_type = "fuzzy (phrase)"
|
| 308 |
|
| 309 |
+
# Step 2.5: if phrase fuzzy failed on original column, try other TEXT columns
|
| 310 |
+
if not sql_rows:
|
| 311 |
+
alt = _try_alternate_columns(fuzzy or sql_query, cfg)
|
| 312 |
+
if alt:
|
| 313 |
+
sql_rows, effective_query = alt
|
| 314 |
+
match_type = "fuzzy (alt column)"
|
| 315 |
+
|
| 316 |
# Step 3: if still no results, try word-level fuzzy (LIKE '%Korea%')
|
| 317 |
if not sql_rows:
|
| 318 |
word_fuzzy = make_fuzzy_query(sql_query, word_level=True)
|