Jakub Claude Opus 4.6 commited on
Commit
fe0d1b4
·
1 Parent(s): 7bd9057

Add fuzzy near-match suggestions for numeric columns

Browse files

When unmatched rows exist and numeric columns have a configured margin,
find candidate rows from the other side where non-numeric columns match
exactly and numeric values differ by at most the margin. Displayed in
CLI output and a new Streamlit "Near matches" tab.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

src/table_comparator/app.py CHANGED
@@ -141,6 +141,7 @@ def main() -> None: # noqa: C901 – kept as single function for Streamlit flow
141
 
142
  fmt_a = fmt_b = None
143
  rounding = None
 
144
  case_sensitive = True
145
  decimal_sep_a = "."
146
  decimal_sep_b = "."
@@ -168,6 +169,18 @@ def main() -> None: # noqa: C901 – kept as single function for Streamlit flow
168
  key=f"rd_{i}",
169
  )
170
 
 
 
 
 
 
 
 
 
 
 
 
 
171
  # Decimal separator controls
172
  st.caption(
173
  "Decimal separator: Specify which character is used as the "
@@ -208,6 +221,7 @@ def main() -> None: # noqa: C901 – kept as single function for Streamlit flow
208
  case_sensitive=case_sensitive,
209
  decimal_separator_a=decimal_sep_a,
210
  decimal_separator_b=decimal_sep_b,
 
211
  )
212
  )
213
 
@@ -251,8 +265,8 @@ def main() -> None: # noqa: C901 – kept as single function for Streamlit flow
251
  m4.metric("B not in A", result.unmatch_count_b)
252
 
253
  # ── result tables ──────────────────────────────────────────────────────
254
- tab_summary, tab_a_miss, tab_b_miss, tab_all_a, tab_all_b = st.tabs(
255
- ["Summary", "A not in B", "B not in A", "All A rows", "All B rows"]
256
  )
257
 
258
  with tab_summary:
@@ -289,6 +303,27 @@ def main() -> None: # noqa: C901 – kept as single function for Streamlit flow
289
  else:
290
  st.success("All rows from Table B were found in Table A!")
291
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
292
  with tab_all_a:
293
  view_a = result.df_a.copy()
294
  view_a.insert(0, "Row", range(1, len(view_a) + 1))
 
141
 
142
  fmt_a = fmt_b = None
143
  rounding = None
144
+ margin = None
145
  case_sensitive = True
146
  decimal_sep_a = "."
147
  decimal_sep_b = "."
 
169
  key=f"rd_{i}",
170
  )
171
 
172
+ # Near-match margin controls
173
+ mg1, mg2 = st.columns(2)
174
+ with mg1:
175
+ do_margin = st.toggle("Near-match margin", key=f"mg_{i}")
176
+ with mg2:
177
+ if do_margin:
178
+ margin = st.number_input(
179
+ "Margin (\u00b1)",
180
+ min_value=0.0, value=5.0,
181
+ key=f"mgv_{i}",
182
+ )
183
+
184
  # Decimal separator controls
185
  st.caption(
186
  "Decimal separator: Specify which character is used as the "
 
221
  case_sensitive=case_sensitive,
222
  decimal_separator_a=decimal_sep_a,
223
  decimal_separator_b=decimal_sep_b,
224
+ margin=margin,
225
  )
226
  )
227
 
 
265
  m4.metric("B not in A", result.unmatch_count_b)
266
 
267
  # ── result tables ──────────────────────────────────────────────────────
268
+ tab_summary, tab_a_miss, tab_b_miss, tab_near, tab_all_a, tab_all_b = st.tabs(
269
+ ["Summary", "A not in B", "B not in A", "Near matches", "All A rows", "All B rows"]
270
  )
271
 
272
  with tab_summary:
 
303
  else:
304
  st.success("All rows from Table B were found in Table A!")
305
 
306
+ with tab_near:
307
+ if not any(m.margin is not None for m in config.columns):
308
+ st.info("No margins configured. Enable a near-match margin on a number column to see suggestions.")
309
+ elif not result.near_matches:
310
+ st.info("No near-matches found.")
311
+ else:
312
+ rows = []
313
+ for nm in result.near_matches:
314
+ row: dict[str, object] = {"Row A": nm.row_a}
315
+ for m in config.columns:
316
+ row[f"A: {m.column_a}"] = result.df_a.at[nm.row_a - 1, m.column_a]
317
+ row["Row B"] = nm.row_b
318
+ for m in config.columns:
319
+ row[f"B: {m.column_b}"] = result.df_b.at[nm.row_b - 1, m.column_b]
320
+ for col, diff in nm.diffs.items():
321
+ row[f"Diff: {col}"] = diff
322
+ rows.append(row)
323
+ st.dataframe(
324
+ pd.DataFrame(rows), use_container_width=True, hide_index=True
325
+ )
326
+
327
  with tab_all_a:
328
  view_a = result.df_a.copy()
329
  view_a.insert(0, "Row", range(1, len(view_a) + 1))
src/table_comparator/cli.py CHANGED
@@ -28,7 +28,7 @@ from pathlib import Path
28
  import click
29
  import pandas as pd
30
 
31
- from table_comparator.comparator import ComparisonResult, compare
32
  from table_comparator.config import (
33
  DATE_FORMAT_EXAMPLES,
34
  VALID_TYPES,
@@ -96,6 +96,27 @@ def _print_results(result: ComparisonResult, *, verbose: bool = False) -> None:
96
  " All rows from Table B were found in Table A.", fg="green", bold=True
97
  )
98
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
99
  if verbose and result.match_count:
100
  click.echo()
101
  click.secho(f" Matched rows from A ({result.match_count}):", bold=True)
 
28
  import click
29
  import pandas as pd
30
 
31
+ from table_comparator.comparator import ComparisonResult, NearMatch, compare
32
  from table_comparator.config import (
33
  DATE_FORMAT_EXAMPLES,
34
  VALID_TYPES,
 
96
  " All rows from Table B were found in Table A.", fg="green", bold=True
97
  )
98
 
99
+ if result.near_matches:
100
+ click.echo()
101
+ click.secho(
102
+ f" Possible near-matches ({len(result.near_matches)}):", bold=True
103
+ )
104
+ for nm in result.near_matches:
105
+ # Gather mapped column values from both sides
106
+ vals_a = []
107
+ vals_b = []
108
+ for m in result.config.columns:
109
+ vals_a.append(str(result.df_a.at[nm.row_a - 1, m.column_a]))
110
+ vals_b.append(str(result.df_b.at[nm.row_b - 1, m.column_b]))
111
+ diff_parts = [
112
+ f"{col}: {d:+g}" for col, d in nm.diffs.items()
113
+ ]
114
+ click.echo(
115
+ f" A Row {nm.row_a} ({', '.join(vals_a)})"
116
+ f" <-> B Row {nm.row_b} ({', '.join(vals_b)})"
117
+ f" [{', '.join(diff_parts)}]"
118
+ )
119
+
120
  if verbose and result.match_count:
121
  click.echo()
122
  click.secho(f" Matched rows from A ({result.match_count}):", bold=True)
src/table_comparator/comparator.py CHANGED
@@ -18,8 +18,10 @@ Algorithm
18
 
19
  from __future__ import annotations
20
 
 
21
  from dataclasses import dataclass, field
22
 
 
23
  import pandas as pd
24
 
25
  from table_comparator.config import ColumnMapping, ComparisonConfig
@@ -37,6 +39,15 @@ _KEY_SEP = "\x00\x00"
37
  # ---------------------------------------------------------------------------
38
 
39
 
 
 
 
 
 
 
 
 
 
40
  def _with_row_number(df: pd.DataFrame, mask: pd.Series) -> pd.DataFrame:
41
  """Return filtered rows with a leading 1-based ``Row`` column."""
42
  out = df[mask].copy()
@@ -54,6 +65,7 @@ class ComparisonResult:
54
  matched_mask_b: pd.Series # boolean; True → row exists in A
55
  config: ComparisonConfig
56
  warnings: list[str] = field(default_factory=list)
 
57
 
58
  # ------------------------------------------------------------------
59
  # A-side convenience views
@@ -203,6 +215,121 @@ def _to_key_str(series: pd.Series) -> pd.Series:
203
  return result
204
 
205
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
206
  # ---------------------------------------------------------------------------
207
  # Public API
208
  # ---------------------------------------------------------------------------
@@ -240,6 +367,8 @@ def compare(
240
  warnings: list[str] = []
241
  key_parts_a: list[pd.Series] = []
242
  key_parts_b: list[pd.Series] = []
 
 
243
 
244
  for m in config.columns:
245
  orig_a = df_a[m.column_a]
@@ -297,6 +426,8 @@ def compare(
297
  warning += f"\n ... and {extra_b - max_examples} more"
298
  warnings.append(warning)
299
 
 
 
300
  key_parts_a.append(_to_key_str(norm_a))
301
  key_parts_b.append(_to_key_str(norm_b))
302
 
@@ -322,6 +453,10 @@ def compare(
322
  cumcount_b = key_b.groupby(key_b).cumcount()
323
  matched_mask_b = cumcount_b < key_b.map(count_a).fillna(0).astype(int)
324
 
 
 
 
 
325
  return ComparisonResult(
326
  df_a=df_a,
327
  matched_mask=matched_mask,
@@ -329,4 +464,5 @@ def compare(
329
  matched_mask_b=matched_mask_b,
330
  config=config,
331
  warnings=warnings,
 
332
  )
 
18
 
19
  from __future__ import annotations
20
 
21
+ from collections import defaultdict
22
  from dataclasses import dataclass, field
23
 
24
+ import numpy as np
25
  import pandas as pd
26
 
27
  from table_comparator.config import ColumnMapping, ComparisonConfig
 
39
  # ---------------------------------------------------------------------------
40
 
41
 
42
+ @dataclass
43
+ class NearMatch:
44
+ """A candidate near-match between an unmatched A row and an unmatched B row."""
45
+
46
+ row_a: int # 1-based row number in A
47
+ row_b: int # 1-based row number in B
48
+ diffs: dict[str, float] # column_a name -> signed difference (a_val - b_val)
49
+
50
+
51
  def _with_row_number(df: pd.DataFrame, mask: pd.Series) -> pd.DataFrame:
52
  """Return filtered rows with a leading 1-based ``Row`` column."""
53
  out = df[mask].copy()
 
65
  matched_mask_b: pd.Series # boolean; True → row exists in A
66
  config: ComparisonConfig
67
  warnings: list[str] = field(default_factory=list)
68
+ near_matches: list[NearMatch] = field(default_factory=list)
69
 
70
  # ------------------------------------------------------------------
71
  # A-side convenience views
 
215
  return result
216
 
217
 
218
+ # ---------------------------------------------------------------------------
219
+ # Near-match search
220
+ # ---------------------------------------------------------------------------
221
+
222
+
223
+ def _find_near_matches(
224
+ config: ComparisonConfig,
225
+ norm_columns_a: dict[str, pd.Series],
226
+ norm_columns_b: dict[str, pd.Series],
227
+ matched_mask: pd.Series,
228
+ matched_mask_b: pd.Series,
229
+ ) -> list[NearMatch]:
230
+ """Find candidate near-matches among unmatched rows.
231
+
232
+ For each unmatched A row, find all unmatched B rows where:
233
+ - all non-numeric mapped columns match exactly (using normalised values)
234
+ - every numeric column with a margin has abs(diff) <= margin
235
+
236
+ Returns a list of NearMatch objects.
237
+ """
238
+ # Identify which number columns have a margin configured
239
+ margin_mappings: list[ColumnMapping] = [
240
+ m for m in config.columns
241
+ if m.col_type == "number" and m.margin is not None
242
+ ]
243
+ if not margin_mappings:
244
+ return []
245
+
246
+ # Non-numeric mappings (must match exactly) + numeric without margin
247
+ exact_mappings: list[ColumnMapping] = [
248
+ m for m in config.columns
249
+ if not (m.col_type == "number" and m.margin is not None)
250
+ ]
251
+
252
+ # Get unmatched indices
253
+ unmatched_idx_a = matched_mask[~matched_mask].index.tolist()
254
+ unmatched_idx_b = matched_mask_b[~matched_mask_b].index.tolist()
255
+
256
+ if not unmatched_idx_a or not unmatched_idx_b:
257
+ return []
258
+
259
+ # Build composite exact-match key for grouping (non-margin columns)
260
+ if exact_mappings:
261
+ exact_parts_a = []
262
+ exact_parts_b = []
263
+ for m in exact_mappings:
264
+ exact_parts_a.append(
265
+ _to_key_str(norm_columns_a[m.column_a]).loc[unmatched_idx_a]
266
+ )
267
+ exact_parts_b.append(
268
+ _to_key_str(norm_columns_b[m.column_b]).loc[unmatched_idx_b]
269
+ )
270
+
271
+ exact_key_a = exact_parts_a[0]
272
+ for part in exact_parts_a[1:]:
273
+ exact_key_a = exact_key_a + _KEY_SEP + part
274
+
275
+ exact_key_b = exact_parts_b[0]
276
+ for part in exact_parts_b[1:]:
277
+ exact_key_b = exact_key_b + _KEY_SEP + part
278
+
279
+ # Group unmatched B rows by their exact key
280
+ b_groups: dict[str, list[int]] = defaultdict(list)
281
+ for idx in unmatched_idx_b:
282
+ b_groups[exact_key_b.loc[idx]].append(idx)
283
+ else:
284
+ # All columns are numeric with margin — every unmatched B is a candidate
285
+ exact_key_a = None
286
+ b_groups = None
287
+
288
+ # Extract numeric values for margin columns
289
+ num_vals_a = {
290
+ m.column_a: norm_columns_a[m.column_a].loc[unmatched_idx_a]
291
+ for m in margin_mappings
292
+ }
293
+ num_vals_b = {
294
+ m.column_a: norm_columns_b[m.column_b].loc[unmatched_idx_b]
295
+ for m in margin_mappings
296
+ }
297
+
298
+ results: list[NearMatch] = []
299
+
300
+ for idx_a in unmatched_idx_a:
301
+ # Get candidate B indices
302
+ if b_groups is not None:
303
+ key = exact_key_a.loc[idx_a]
304
+ candidates = b_groups.get(key, [])
305
+ else:
306
+ candidates = unmatched_idx_b
307
+
308
+ for idx_b in candidates:
309
+ diffs: dict[str, float] = {}
310
+ match = True
311
+ for m in margin_mappings:
312
+ val_a = num_vals_a[m.column_a].loc[idx_a]
313
+ val_b = num_vals_b[m.column_a].loc[idx_b]
314
+ # If either value is NaN, can't compute a numeric diff
315
+ if np.isnan(val_a) or np.isnan(val_b):
316
+ match = False
317
+ break
318
+ diff = val_a - val_b
319
+ if abs(diff) > m.margin:
320
+ match = False
321
+ break
322
+ diffs[m.column_a] = diff
323
+ if match:
324
+ results.append(NearMatch(
325
+ row_a=idx_a + 1, # 1-based
326
+ row_b=idx_b + 1, # 1-based
327
+ diffs=diffs,
328
+ ))
329
+
330
+ return results
331
+
332
+
333
  # ---------------------------------------------------------------------------
334
  # Public API
335
  # ---------------------------------------------------------------------------
 
367
  warnings: list[str] = []
368
  key_parts_a: list[pd.Series] = []
369
  key_parts_b: list[pd.Series] = []
370
+ norm_columns_a: dict[str, pd.Series] = {}
371
+ norm_columns_b: dict[str, pd.Series] = {}
372
 
373
  for m in config.columns:
374
  orig_a = df_a[m.column_a]
 
426
  warning += f"\n ... and {extra_b - max_examples} more"
427
  warnings.append(warning)
428
 
429
+ norm_columns_a[m.column_a] = norm_a
430
+ norm_columns_b[m.column_b] = norm_b
431
  key_parts_a.append(_to_key_str(norm_a))
432
  key_parts_b.append(_to_key_str(norm_b))
433
 
 
453
  cumcount_b = key_b.groupby(key_b).cumcount()
454
  matched_mask_b = cumcount_b < key_b.map(count_a).fillna(0).astype(int)
455
 
456
+ near_matches = _find_near_matches(
457
+ config, norm_columns_a, norm_columns_b, matched_mask, matched_mask_b,
458
+ )
459
+
460
  return ComparisonResult(
461
  df_a=df_a,
462
  matched_mask=matched_mask,
 
464
  matched_mask_b=matched_mask_b,
465
  config=config,
466
  warnings=warnings,
467
+ near_matches=near_matches,
468
  )
src/table_comparator/config.py CHANGED
@@ -37,6 +37,7 @@ class ColumnMapping:
37
  case_sensitive: bool = True # text only
38
  decimal_separator_a: str = "." # "." or "," for Table A (number only)
39
  decimal_separator_b: str = "." # "." or "," for Table B (number only)
 
40
 
41
  # ------------------------------------------------------------------
42
  # validation
@@ -70,6 +71,13 @@ class ColumnMapping:
70
  f"decimal_separator_b for '{self.column_b}' must be '.' or ',', "
71
  f"got '{self.decimal_separator_b}'"
72
  )
 
 
 
 
 
 
 
73
  return errors
74
 
75
  # ------------------------------------------------------------------
 
37
  case_sensitive: bool = True # text only
38
  decimal_separator_a: str = "." # "." or "," for Table A (number only)
39
  decimal_separator_b: str = "." # "." or "," for Table B (number only)
40
+ margin: float | None = None # absolute tolerance for near-match (number only)
41
 
42
  # ------------------------------------------------------------------
43
  # validation
 
71
  f"decimal_separator_b for '{self.column_b}' must be '.' or ',', "
72
  f"got '{self.decimal_separator_b}'"
73
  )
74
+ if self.margin is not None and self.margin < 0:
75
+ errors.append(f"Margin for '{self.column_a}' must be >= 0")
76
+ if self.margin is not None and self.col_type != "number":
77
+ errors.append(
78
+ f"Margin is only valid for number columns, "
79
+ f"but '{self.column_a}' has type '{self.col_type}'"
80
+ )
81
  return errors
82
 
83
  # ------------------------------------------------------------------
tests/test_comparator.py CHANGED
@@ -5,7 +5,7 @@ from pathlib import Path
5
  import pandas as pd
6
  import pytest
7
 
8
- from table_comparator.comparator import compare
9
  from table_comparator.config import ColumnMapping, ComparisonConfig
10
 
11
 
@@ -836,3 +836,239 @@ class TestBidirectional:
836
  assert result.total_b == 0
837
  assert result.match_count == 0
838
  assert result.match_count_b == 0
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
5
  import pandas as pd
6
  import pytest
7
 
8
+ from table_comparator.comparator import NearMatch, compare
9
  from table_comparator.config import ColumnMapping, ComparisonConfig
10
 
11
 
 
836
  assert result.total_b == 0
837
  assert result.match_count == 0
838
  assert result.match_count_b == 0
839
+
840
+
841
+ # ---------------------------------------------------------------------------
842
+ # near-match suggestions
843
+ # ---------------------------------------------------------------------------
844
+
845
+
846
+ class TestNearMatches:
847
+ def test_no_margin_configured(self):
848
+ """Without margin, near_matches is empty."""
849
+ a = pd.DataFrame({"name": ["Alice"], "val": [150.25]})
850
+ b = pd.DataFrame({"name": ["Alice"], "val": [150.00]})
851
+ result = compare(
852
+ _cfg([
853
+ ColumnMapping(column_a="name", column_b="name", col_type="text"),
854
+ ColumnMapping(column_a="val", column_b="val", col_type="number"),
855
+ ]),
856
+ a, b,
857
+ )
858
+ assert result.near_matches == []
859
+
860
+ def test_margin_finds_candidate(self):
861
+ """A row within margin is reported as near-match."""
862
+ a = pd.DataFrame({"name": ["Alice"], "val": [150.25]})
863
+ b = pd.DataFrame({"name": ["Alice"], "val": [150.00]})
864
+ result = compare(
865
+ _cfg([
866
+ ColumnMapping(column_a="name", column_b="name", col_type="text"),
867
+ ColumnMapping(column_a="val", column_b="val", col_type="number", margin=1.0),
868
+ ]),
869
+ a, b,
870
+ )
871
+ assert len(result.near_matches) == 1
872
+ nm = result.near_matches[0]
873
+ assert nm.row_a == 1
874
+ assert nm.row_b == 1
875
+ assert nm.diffs == {"val": pytest.approx(0.25)}
876
+
877
+ def test_outside_margin_no_match(self):
878
+ """Difference exceeding margin is not a near-match."""
879
+ a = pd.DataFrame({"name": ["Alice"], "val": [150.25]})
880
+ b = pd.DataFrame({"name": ["Alice"], "val": [150.00]})
881
+ result = compare(
882
+ _cfg([
883
+ ColumnMapping(column_a="name", column_b="name", col_type="text"),
884
+ ColumnMapping(column_a="val", column_b="val", col_type="number", margin=0.1),
885
+ ]),
886
+ a, b,
887
+ )
888
+ assert result.near_matches == []
889
+
890
+ def test_exact_matched_rows_excluded(self):
891
+ """Rows that already exact-match are not in the near-match pool."""
892
+ a = pd.DataFrame({"name": ["Alice", "Bob"], "val": [100.0, 150.25]})
893
+ b = pd.DataFrame({"name": ["Alice", "Bob"], "val": [100.0, 150.00]})
894
+ result = compare(
895
+ _cfg([
896
+ ColumnMapping(column_a="name", column_b="name", col_type="text"),
897
+ ColumnMapping(column_a="val", column_b="val", col_type="number", margin=1.0),
898
+ ]),
899
+ a, b,
900
+ )
901
+ # Alice exact-matches; Bob is near-match
902
+ assert result.match_count == 1
903
+ assert len(result.near_matches) == 1
904
+ nm = result.near_matches[0]
905
+ assert nm.row_a == 2 # Bob
906
+ assert nm.row_b == 2
907
+
908
+ def test_multiple_candidates(self):
909
+ """One A row can have multiple B candidates."""
910
+ a = pd.DataFrame({"name": ["Alice"], "val": [100.0]})
911
+ b = pd.DataFrame({"name": ["Alice", "Alice"], "val": [99.5, 100.3]})
912
+ result = compare(
913
+ _cfg([
914
+ ColumnMapping(column_a="name", column_b="name", col_type="text"),
915
+ ColumnMapping(column_a="val", column_b="val", col_type="number", margin=1.0),
916
+ ]),
917
+ a, b,
918
+ )
919
+ # None exact-match (name+val composite key differs)
920
+ assert result.match_count == 0
921
+ assert len(result.near_matches) == 2
922
+ b_rows = sorted(nm.row_b for nm in result.near_matches)
923
+ assert b_rows == [1, 2]
924
+
925
+ def test_multiple_numeric_columns_all_within_margin(self):
926
+ """All numeric columns must be within their respective margins."""
927
+ a = pd.DataFrame({"name": ["Alice"], "x": [100.0], "y": [200.0]})
928
+ b = pd.DataFrame({"name": ["Alice"], "x": [100.5], "y": [200.3]})
929
+ result = compare(
930
+ _cfg([
931
+ ColumnMapping(column_a="name", column_b="name", col_type="text"),
932
+ ColumnMapping(column_a="x", column_b="x", col_type="number", margin=1.0),
933
+ ColumnMapping(column_a="y", column_b="y", col_type="number", margin=1.0),
934
+ ]),
935
+ a, b,
936
+ )
937
+ assert len(result.near_matches) == 1
938
+ assert result.near_matches[0].diffs == {
939
+ "x": pytest.approx(-0.5),
940
+ "y": pytest.approx(-0.3),
941
+ }
942
+
943
+ def test_multiple_numeric_columns_one_outside_margin(self):
944
+ """If any numeric column exceeds its margin, no near-match."""
945
+ a = pd.DataFrame({"name": ["Alice"], "x": [100.0], "y": [200.0]})
946
+ b = pd.DataFrame({"name": ["Alice"], "x": [100.5], "y": [205.0]})
947
+ result = compare(
948
+ _cfg([
949
+ ColumnMapping(column_a="name", column_b="name", col_type="text"),
950
+ ColumnMapping(column_a="x", column_b="x", col_type="number", margin=1.0),
951
+ ColumnMapping(column_a="y", column_b="y", col_type="number", margin=1.0),
952
+ ]),
953
+ a, b,
954
+ )
955
+ assert result.near_matches == []
956
+
957
+ def test_non_numeric_columns_must_match_exactly(self):
958
+ """Different text/date values prevent near-match even if number is within margin."""
959
+ a = pd.DataFrame({"name": ["Alice"], "val": [100.0]})
960
+ b = pd.DataFrame({"name": ["Bob"], "val": [100.5]})
961
+ result = compare(
962
+ _cfg([
963
+ ColumnMapping(column_a="name", column_b="name", col_type="text"),
964
+ ColumnMapping(column_a="val", column_b="val", col_type="number", margin=1.0),
965
+ ]),
966
+ a, b,
967
+ )
968
+ assert result.near_matches == []
969
+
970
+ def test_all_columns_numeric(self):
971
+ """When all columns are numeric with margin, cross-check all unmatched pairs."""
972
+ a = pd.DataFrame({"x": [100.0], "y": [200.0]})
973
+ b = pd.DataFrame({"x": [100.5], "y": [200.3]})
974
+ result = compare(
975
+ _cfg([
976
+ ColumnMapping(column_a="x", column_b="x", col_type="number", margin=1.0),
977
+ ColumnMapping(column_a="y", column_b="y", col_type="number", margin=1.0),
978
+ ]),
979
+ a, b,
980
+ )
981
+ assert len(result.near_matches) == 1
982
+
983
+ def test_margin_with_rounding_interaction(self):
984
+ """Margin applies to already-rounded values."""
985
+ a = pd.DataFrame({"name": ["Alice"], "val": [150.254]})
986
+ b = pd.DataFrame({"name": ["Alice"], "val": [150.004]})
987
+ # rounding=2: 150.254→150.25, 150.004→150.00 → diff = 0.25
988
+ result = compare(
989
+ _cfg([
990
+ ColumnMapping(column_a="name", column_b="name", col_type="text"),
991
+ ColumnMapping(column_a="val", column_b="val", col_type="number", rounding=2, margin=0.3),
992
+ ]),
993
+ a, b,
994
+ )
995
+ assert len(result.near_matches) == 1
996
+ assert result.near_matches[0].diffs["val"] == pytest.approx(0.25)
997
+
998
+ def test_margin_zero_only_exact_diff(self):
999
+ """Margin=0 only matches rows with exactly equal numeric values (already exact-matched)."""
1000
+ a = pd.DataFrame({"name": ["Alice"], "val": [100.0]})
1001
+ b = pd.DataFrame({"name": ["Alice"], "val": [100.01]})
1002
+ result = compare(
1003
+ _cfg([
1004
+ ColumnMapping(column_a="name", column_b="name", col_type="text"),
1005
+ ColumnMapping(column_a="val", column_b="val", col_type="number", margin=0.0),
1006
+ ]),
1007
+ a, b,
1008
+ )
1009
+ assert result.near_matches == []
1010
+
1011
+ def test_nan_in_margin_column_no_near_match(self):
1012
+ """NaN in a margin column cannot produce a near-match."""
1013
+ a = pd.DataFrame({"name": ["Alice"], "val": [float("nan")]})
1014
+ b = pd.DataFrame({"name": ["Alice"], "val": [100.0]})
1015
+ result = compare(
1016
+ _cfg([
1017
+ ColumnMapping(column_a="name", column_b="name", col_type="text"),
1018
+ ColumnMapping(column_a="val", column_b="val", col_type="number", margin=1000.0),
1019
+ ]),
1020
+ a, b,
1021
+ )
1022
+ assert result.near_matches == []
1023
+
1024
+ def test_sample_data_margin(self, df_a, df_b):
1025
+ """David (150.25 vs 150.00) and Grace (90.75 vs 91.00) appear as near-matches."""
1026
+ cfg = ComparisonConfig(columns=[
1027
+ ColumnMapping(column_a="Name", column_b="Customer", col_type="text"),
1028
+ ColumnMapping(
1029
+ column_a="Transaction Date", column_b="Date", col_type="date",
1030
+ date_format_a="%d/%m/%Y", date_format_b="%Y-%m-%d",
1031
+ ),
1032
+ ColumnMapping(
1033
+ column_a="Amount", column_b="Value", col_type="number",
1034
+ rounding=2, margin=0.5,
1035
+ ),
1036
+ ])
1037
+ result = compare(cfg, df_a, df_b)
1038
+ # David and Grace should be near-matches
1039
+ near_a_rows = {nm.row_a for nm in result.near_matches}
1040
+ # Find David and Grace's row numbers (1-based)
1041
+ names_a = df_a["Name"].tolist()
1042
+ david_row = names_a.index("David") + 1
1043
+ grace_row = names_a.index("Grace") + 1
1044
+ assert david_row in near_a_rows
1045
+ assert grace_row in near_a_rows
1046
+
1047
+ def test_numeric_column_without_margin_treated_as_exact(self):
1048
+ """A number column without margin is treated as exact-match requirement."""
1049
+ a = pd.DataFrame({"x": [100.0], "y": [200.0]})
1050
+ b = pd.DataFrame({"x": [100.5], "y": [200.0]})
1051
+ # x has margin, y does not → y must match exactly for near-match
1052
+ result = compare(
1053
+ _cfg([
1054
+ ColumnMapping(column_a="x", column_b="x", col_type="number", margin=1.0),
1055
+ ColumnMapping(column_a="y", column_b="y", col_type="number"),
1056
+ ]),
1057
+ a, b,
1058
+ )
1059
+ # y matches exactly, x is within margin → near-match
1060
+ assert len(result.near_matches) == 1
1061
+
1062
+ def test_numeric_column_without_margin_blocks_near_match(self):
1063
+ """A number column without margin that doesn't match blocks near-match."""
1064
+ a = pd.DataFrame({"x": [100.0], "y": [200.0]})
1065
+ b = pd.DataFrame({"x": [100.5], "y": [200.5]})
1066
+ result = compare(
1067
+ _cfg([
1068
+ ColumnMapping(column_a="x", column_b="x", col_type="number", margin=1.0),
1069
+ ColumnMapping(column_a="y", column_b="y", col_type="number"),
1070
+ ]),
1071
+ a, b,
1072
+ )
1073
+ # y doesn't match exactly → no near-match
1074
+ assert result.near_matches == []