Spaces:
Sleeping
Sleeping
Commit ·
d4a1a18
1
Parent(s): 6b689a5
fix: add word-level fuzzy SQL matching for country name variants
Browse filesCountry names like Korea, Republic of do not contain South Korea as a
substring, so phrase-level LIKE fails. Word-level fuzzy extracts significant
content words (dropping directional/geopolitical qualifiers) and matches
individually: WHERE Country LIKE %Korea%.
Three-step strategy: exact -> phrase fuzzy -> word-level fuzzy.
Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
- src/retriever.py +11 -2
- src/sql_retriever.py +47 -6
src/retriever.py
CHANGED
|
@@ -225,14 +225,23 @@ def _run_sql_retrieval(sql_query: str, cfg: dict) -> tuple[list[dict], str, str]
|
|
| 225 |
effective_query = sql_query
|
| 226 |
match_type = "exact"
|
| 227 |
|
| 228 |
-
# Step 2: if no results, try fuzzy
|
| 229 |
if not sql_rows:
|
| 230 |
fuzzy = make_fuzzy_query(sql_query)
|
| 231 |
if fuzzy:
|
| 232 |
sql_rows = execute_sql_query(fuzzy, cfg)
|
| 233 |
if sql_rows:
|
| 234 |
effective_query = fuzzy
|
| 235 |
-
match_type = "fuzzy"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 236 |
|
| 237 |
if not sql_rows:
|
| 238 |
return [], "", ""
|
|
|
|
| 225 |
effective_query = sql_query
|
| 226 |
match_type = "exact"
|
| 227 |
|
| 228 |
+
# Step 2: if no results, try phrase-level fuzzy (LIKE '%South Korea%')
|
| 229 |
if not sql_rows:
|
| 230 |
fuzzy = make_fuzzy_query(sql_query)
|
| 231 |
if fuzzy:
|
| 232 |
sql_rows = execute_sql_query(fuzzy, cfg)
|
| 233 |
if sql_rows:
|
| 234 |
effective_query = fuzzy
|
| 235 |
+
match_type = "fuzzy (phrase)"
|
| 236 |
+
|
| 237 |
+
# Step 3: if still no results, try word-level fuzzy (LIKE '%Korea%')
|
| 238 |
+
if not sql_rows:
|
| 239 |
+
word_fuzzy = make_fuzzy_query(sql_query, word_level=True)
|
| 240 |
+
if word_fuzzy and word_fuzzy != (make_fuzzy_query(sql_query) or ""):
|
| 241 |
+
sql_rows = execute_sql_query(word_fuzzy, cfg)
|
| 242 |
+
if sql_rows:
|
| 243 |
+
effective_query = word_fuzzy
|
| 244 |
+
match_type = "fuzzy (word)"
|
| 245 |
|
| 246 |
if not sql_rows:
|
| 247 |
return [], "", ""
|
src/sql_retriever.py
CHANGED
|
@@ -154,20 +154,61 @@ def format_sql_results_as_context(
|
|
| 154 |
return "\n".join(parts)
|
| 155 |
|
| 156 |
|
| 157 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 158 |
"""Convert exact equality on text values to LIKE fuzzy matching.
|
| 159 |
|
| 160 |
-
|
| 161 |
-
|
| 162 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 163 |
|
| 164 |
Returns the modified query, or None if no substitutions were made.
|
| 165 |
"""
|
| 166 |
-
# Match patterns like: column = 'value' or column='value'
|
| 167 |
pattern = re.compile(
|
| 168 |
r"""(["']?\w+["']?\s*)=\s*'([^']+)'""",
|
| 169 |
)
|
| 170 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 171 |
if count == 0:
|
| 172 |
return None
|
| 173 |
return new_sql
|
|
|
|
| 154 |
return "\n".join(parts)
|
| 155 |
|
| 156 |
|
| 157 |
+
_FUZZY_STOPWORDS = frozenset({
|
| 158 |
+
"the", "and", "for", "are", "but", "not", "you", "all",
|
| 159 |
+
"can", "had", "her", "was", "one", "our", "out", "has",
|
| 160 |
+
"what", "how", "who", "which", "when", "where", "with",
|
| 161 |
+
"from", "that", "this", "than", "then", "they", "been",
|
| 162 |
+
"south", "north", "east", "west", "new", "old", "united",
|
| 163 |
+
"democratic", "republic", "people", "state", "states",
|
| 164 |
+
"islamic", "federal", "kingdom",
|
| 165 |
+
})
|
| 166 |
+
|
| 167 |
+
|
| 168 |
+
def make_fuzzy_query(sql: str, word_level: bool = False) -> str | None:
|
| 169 |
"""Convert exact equality on text values to LIKE fuzzy matching.
|
| 170 |
|
| 171 |
+
When ``word_level`` is False (default), transforms
|
| 172 |
+
``column = 'value'`` into ``column LIKE '%value%'``
|
| 173 |
+
(phrase-level matching).
|
| 174 |
+
|
| 175 |
+
When ``word_level`` is True, transforms
|
| 176 |
+
``column = 'South Korea'`` into
|
| 177 |
+
``(column LIKE '%Korea%')``
|
| 178 |
+
by extracting significant content words (dropping stopwords and
|
| 179 |
+
directional/geopolitical qualifiers). This handles cases where
|
| 180 |
+
"South Korea" is stored as "Korea, Republic of".
|
| 181 |
|
| 182 |
Returns the modified query, or None if no substitutions were made.
|
| 183 |
"""
|
|
|
|
| 184 |
pattern = re.compile(
|
| 185 |
r"""(["']?\w+["']?\s*)=\s*'([^']+)'""",
|
| 186 |
)
|
| 187 |
+
|
| 188 |
+
if not word_level:
|
| 189 |
+
new_sql, count = pattern.subn(r"\1LIKE '%\2%'", sql)
|
| 190 |
+
if count == 0:
|
| 191 |
+
return None
|
| 192 |
+
return new_sql
|
| 193 |
+
|
| 194 |
+
# Word-level: extract significant words from each value
|
| 195 |
+
def _word_replace(m):
|
| 196 |
+
col_part = m.group(1)
|
| 197 |
+
value = m.group(2)
|
| 198 |
+
words = [
|
| 199 |
+
w for w in re.split(r'\W+', value)
|
| 200 |
+
if len(w) >= 3 and w.lower() not in _FUZZY_STOPWORDS
|
| 201 |
+
]
|
| 202 |
+
if not words:
|
| 203 |
+
# Fallback: use the original value as phrase LIKE
|
| 204 |
+
return f"{col_part}LIKE '%{value}%'"
|
| 205 |
+
if len(words) == 1:
|
| 206 |
+
return f"{col_part}LIKE '%{words[0]}%'"
|
| 207 |
+
# Multiple words: join with AND on the same column
|
| 208 |
+
conditions = [f"{col_part}LIKE '%{w}%'" for w in words]
|
| 209 |
+
return "(" + " AND ".join(conditions) + ")"
|
| 210 |
+
|
| 211 |
+
new_sql, count = pattern.subn(_word_replace, sql)
|
| 212 |
if count == 0:
|
| 213 |
return None
|
| 214 |
return new_sql
|