LIANJie-Jason Claude Opus 4.6 commited on
Commit
d4a1a18
·
1 Parent(s): 6b689a5

fix: add word-level fuzzy SQL matching for country name variants

Browse files

Country names like Korea, Republic of do not contain South Korea as a
substring, so phrase-level LIKE fails. Word-level fuzzy extracts significant
content words (dropping directional/geopolitical qualifiers) and matches
individually: WHERE Country LIKE %Korea%.

Three-step strategy: exact -> phrase fuzzy -> word-level fuzzy.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

Files changed (2) hide show
  1. src/retriever.py +11 -2
  2. src/sql_retriever.py +47 -6
src/retriever.py CHANGED
@@ -225,14 +225,23 @@ def _run_sql_retrieval(sql_query: str, cfg: dict) -> tuple[list[dict], str, str]
225
  effective_query = sql_query
226
  match_type = "exact"
227
 
228
- # Step 2: if no results, try fuzzy fallback
229
  if not sql_rows:
230
  fuzzy = make_fuzzy_query(sql_query)
231
  if fuzzy:
232
  sql_rows = execute_sql_query(fuzzy, cfg)
233
  if sql_rows:
234
  effective_query = fuzzy
235
- match_type = "fuzzy"
 
 
 
 
 
 
 
 
 
236
 
237
  if not sql_rows:
238
  return [], "", ""
 
225
  effective_query = sql_query
226
  match_type = "exact"
227
 
228
+ # Step 2: if no results, try phrase-level fuzzy (LIKE '%South Korea%')
229
  if not sql_rows:
230
  fuzzy = make_fuzzy_query(sql_query)
231
  if fuzzy:
232
  sql_rows = execute_sql_query(fuzzy, cfg)
233
  if sql_rows:
234
  effective_query = fuzzy
235
+ match_type = "fuzzy (phrase)"
236
+
237
+ # Step 3: if still no results, try word-level fuzzy (LIKE '%Korea%')
238
+ if not sql_rows:
239
+ word_fuzzy = make_fuzzy_query(sql_query, word_level=True)
240
+ if word_fuzzy and word_fuzzy != (make_fuzzy_query(sql_query) or ""):
241
+ sql_rows = execute_sql_query(word_fuzzy, cfg)
242
+ if sql_rows:
243
+ effective_query = word_fuzzy
244
+ match_type = "fuzzy (word)"
245
 
246
  if not sql_rows:
247
  return [], "", ""
src/sql_retriever.py CHANGED
@@ -154,20 +154,61 @@ def format_sql_results_as_context(
154
  return "\n".join(parts)
155
 
156
 
157
- def make_fuzzy_query(sql: str) -> str | None:
 
 
 
 
 
 
 
 
 
 
 
158
  """Convert exact equality on text values to LIKE fuzzy matching.
159
 
160
- Transforms ``column = 'value'`` into ``column LIKE '%value%'``
161
- so that partial country names (e.g. "South Korea" matching
162
- "South Korea (Republic of Korea)") can be found.
 
 
 
 
 
 
 
163
 
164
  Returns the modified query, or None if no substitutions were made.
165
  """
166
- # Match patterns like: column = 'value' or column='value'
167
  pattern = re.compile(
168
  r"""(["']?\w+["']?\s*)=\s*'([^']+)'""",
169
  )
170
- new_sql, count = pattern.subn(r"\1LIKE '%\2%'", sql)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
171
  if count == 0:
172
  return None
173
  return new_sql
 
154
  return "\n".join(parts)
155
 
156
 
157
+ _FUZZY_STOPWORDS = frozenset({
158
+ "the", "and", "for", "are", "but", "not", "you", "all",
159
+ "can", "had", "her", "was", "one", "our", "out", "has",
160
+ "what", "how", "who", "which", "when", "where", "with",
161
+ "from", "that", "this", "than", "then", "they", "been",
162
+ "south", "north", "east", "west", "new", "old", "united",
163
+ "democratic", "republic", "people", "state", "states",
164
+ "islamic", "federal", "kingdom",
165
+ })
166
+
167
+
168
+ def make_fuzzy_query(sql: str, word_level: bool = False) -> str | None:
169
  """Convert exact equality on text values to LIKE fuzzy matching.
170
 
171
+ When ``word_level`` is False (default), transforms
172
+ ``column = 'value'`` into ``column LIKE '%value%'``
173
+ (phrase-level matching).
174
+
175
+ When ``word_level`` is True, transforms
176
+ ``column = 'South Korea'`` into
177
+ ``(column LIKE '%Korea%')``
178
+ by extracting significant content words (dropping stopwords and
179
+ directional/geopolitical qualifiers). This handles cases where
180
+ "South Korea" is stored as "Korea, Republic of".
181
 
182
  Returns the modified query, or None if no substitutions were made.
183
  """
 
184
  pattern = re.compile(
185
  r"""(["']?\w+["']?\s*)=\s*'([^']+)'""",
186
  )
187
+
188
+ if not word_level:
189
+ new_sql, count = pattern.subn(r"\1LIKE '%\2%'", sql)
190
+ if count == 0:
191
+ return None
192
+ return new_sql
193
+
194
+ # Word-level: extract significant words from each value
195
+ def _word_replace(m):
196
+ col_part = m.group(1)
197
+ value = m.group(2)
198
+ words = [
199
+ w for w in re.split(r'\W+', value)
200
+ if len(w) >= 3 and w.lower() not in _FUZZY_STOPWORDS
201
+ ]
202
+ if not words:
203
+ # Fallback: use the original value as phrase LIKE
204
+ return f"{col_part}LIKE '%{value}%'"
205
+ if len(words) == 1:
206
+ return f"{col_part}LIKE '%{words[0]}%'"
207
+ # Multiple words: join with AND on the same column
208
+ conditions = [f"{col_part}LIKE '%{w}%'" for w in words]
209
+ return "(" + " AND ".join(conditions) + ")"
210
+
211
+ new_sql, count = pattern.subn(_word_replace, sql)
212
  if count == 0:
213
  return None
214
  return new_sql