Fix four defects found reviewing the matcher
Browse filesAll four reproduce on the customer's August file and three of them are
visible in what the customer reads.
Roll-up matches reported the first ERP entry's amount and line count
rather than the group's, so every roll-up rendered as a matched pair
whose two sides disagreed β "ERP 36,000.00 / Bank -438,623.89" for a
seven-entry roll-up. All 8 August and 27 July roll-ups now tie.
A reversal carries the same magnitude as the payment it backs out, and
candidates are keyed on magnitude, so a payment could be matched to its
own reversal β August holds 539,249.00 as two debits and one reversal
credit, and ranking on date alone could hand an ERP payment the credit.
That reports the reversal as cleared and leaves a real payment
outstanding. Reversals are now flagged and ranked last, so one is only
taken when nothing else fits. The ERP sign is not usable as a direction
check here: across the 132 reference-matched pairs the ERP side is
positive 101 times and negative 31, while the bank side is always a
debit.
_name_close compared the ERP payee against the whole bank description,
including "Payment Instruction", the reference number and the channel
suffix, which diluted the ratio below the threshold even for obvious
pairs β 'ACWORTH RECYCLING LL' against 'Acworth Recycling dba Payment
Instruction 1923-039-116-08' returned False. It now compares against the
payee portion and accepts containment in either direction, which also
covers the 27-char ERP truncation.
Unmatched totals summed signed amounts, so unmatched reversal credits
cancelled unmatched debits: August reported -$8,033,687.59 outstanding
where the gross is $9,764,417.37. Both figures are now gross and labelled
as such, including the one handed to the narrative prompt.
Coverage is unchanged at 91.6% ERP / 86.7% bank for August and 78.3% /
70.6% for July.
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
- app_chatgpt.py +65 -16
|
@@ -391,13 +391,35 @@ def _norm_name(text):
|
|
| 391 |
return re.sub(r"\s+", " ", s).strip()
|
| 392 |
|
| 393 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 394 |
def _name_close(a, b):
|
| 395 |
-
"""True when two payee strings plausibly denote the same vendor.
|
| 396 |
-
|
| 397 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 398 |
if not a or not b:
|
| 399 |
return False
|
| 400 |
-
if a
|
| 401 |
return True
|
| 402 |
return difflib.SequenceMatcher(None, a, b).ratio() >= 0.82
|
| 403 |
|
|
@@ -590,6 +612,11 @@ def normalize_bank_statement(df):
|
|
| 590 |
})
|
| 591 |
out = out[out["amount"].round(2) != 0].reset_index(drop=True)
|
| 592 |
out["refs"] = out["desc"].map(lambda s: set(_REF_RE.findall(s)))
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 593 |
|
| 594 |
n_ref = int(out["refs"].map(bool).sum())
|
| 595 |
logger.info(f"π¦ Bank normalised: {len(out)} transaction(s), "
|
|
@@ -631,15 +658,23 @@ def match_transactions(erp, bank, amount_tol=0.01, date_tol_days=45):
|
|
| 631 |
matches = []
|
| 632 |
|
| 633 |
def take(ei, bi, method, confidence, erp_rows=None):
|
|
|
|
|
|
|
|
|
|
|
|
|
| 634 |
e, b = erp.loc[ei], bank.loc[bi]
|
|
|
|
| 635 |
matches.append({
|
| 636 |
-
"erp_entry":
|
| 637 |
-
"
|
|
|
|
|
|
|
|
|
|
| 638 |
"bank_date": b["date"], "bank_desc": b["desc"], "bank_amount": b["amount"],
|
| 639 |
"method": method, "confidence": confidence,
|
| 640 |
-
"erp_rows":
|
| 641 |
})
|
| 642 |
-
for idx in
|
| 643 |
erp_open.discard(idx)
|
| 644 |
bank_open.discard(bi)
|
| 645 |
|
|
@@ -663,11 +698,23 @@ def match_transactions(erp, bank, amount_tol=0.01, date_tol_days=45):
|
|
| 663 |
gap = day_gap(ei, bi)
|
| 664 |
return True if gap is None else gap <= days
|
| 665 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 666 |
def nearest(ei, options):
|
| 667 |
-
"""Of several equal-amount bank lines, the
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 668 |
if not options:
|
| 669 |
return None
|
| 670 |
-
return min(options, key=lambda bi: (
|
|
|
|
| 671 |
day_gap(ei, bi) or 0))
|
| 672 |
|
| 673 |
# ββ Pass 1: the bank's own reference to the ERP entry ββββββββββββββββββββ
|
|
@@ -1255,8 +1302,8 @@ FINAL FIGURES
|
|
| 1255 |
- ERP transactions (distribution lines rolled up to entries): {stats['erp_count']} (from {stats['erp_lines']} detail lines)
|
| 1256 |
- {stmt_label} transactions: {stats['bank_count']}
|
| 1257 |
- Matched: {stats['matched']} ({stats['erp_cov']:.1f}% of ERP, {stats['bank_cov']:.1f}% of {stmt_label})
|
| 1258 |
-
- Unmatched in ERP: {stats['erp_unmatched']} worth {_fmt_money(stats['erp_unmatched_value'])}
|
| 1259 |
-
- Unmatched in {stmt_label}: {stats['bank_unmatched']} worth {_fmt_money(stats['bank_unmatched_value'])}
|
| 1260 |
- Match methods used: {stats['methods']}
|
| 1261 |
|
| 1262 |
LARGEST UNMATCHED {stmt_label.upper()} ITEMS
|
|
@@ -1339,9 +1386,9 @@ def render_reconciliation_markdown(matched, erp_un, bank_un, stats, stmt_label,
|
|
| 1339 |
parts.append(f"| ERP coverage | {stats['erp_cov']:.1f}% |")
|
| 1340 |
parts.append(f"| {stmt_label} coverage | {stats['bank_cov']:.1f}% |")
|
| 1341 |
parts.append(f"| Unmatched in ERP | {stats['erp_unmatched']:,} "
|
| 1342 |
-
f"({_fmt_money(stats['erp_unmatched_value'])}) |")
|
| 1343 |
parts.append(f"| Unmatched in {stmt_label} | {stats['bank_unmatched']:,} "
|
| 1344 |
-
f"({_fmt_money(stats['bank_unmatched_value'])}) |")
|
| 1345 |
parts.append("")
|
| 1346 |
|
| 1347 |
if steps:
|
|
@@ -1414,8 +1461,10 @@ def reconcile_deterministic(erp_df, external_df, stmt_label, extraction_warning=
|
|
| 1414 |
"bank_cov": (len(bank) - len(bank_un)) / len(bank) * 100 if len(bank) else 0.0,
|
| 1415 |
"erp_unmatched": len(erp_un),
|
| 1416 |
"bank_unmatched": len(bank_un),
|
| 1417 |
-
|
| 1418 |
-
|
|
|
|
|
|
|
| 1419 |
"methods": methods,
|
| 1420 |
}
|
| 1421 |
|
|
|
|
| 391 |
return re.sub(r"\s+", " ", s).strip()
|
| 392 |
|
| 393 |
|
| 394 |
+
# Payment-rail boilerplate. Bank descriptions read "Acworth Recycling dba
|
| 395 |
+
# Payment Instruction 1923-039-116-08 - Acworth Recycling dba Atlas Demolition
|
| 396 |
+
# - ach"; only the text before the first marker is the payee.
|
| 397 |
+
_RAIL_RE = re.compile(
|
| 398 |
+
r"\b(payment instruction|reversal|ref\.|wire|eft|trace|memo|ach|check)\b", re.I)
|
| 399 |
+
|
| 400 |
+
|
| 401 |
+
def _bank_payee(description):
|
| 402 |
+
"""The payee portion of a bank description, minus the rail boilerplate."""
|
| 403 |
+
text = _text(description)
|
| 404 |
+
m = _RAIL_RE.search(text)
|
| 405 |
+
head = text[:m.start()] if m else text
|
| 406 |
+
return head.strip(" -\u2013\u2014|,") or text
|
| 407 |
+
|
| 408 |
+
|
| 409 |
def _name_close(a, b):
|
| 410 |
+
"""True when two payee strings plausibly denote the same vendor.
|
| 411 |
+
|
| 412 |
+
`b` is matched against its payee portion rather than the whole bank
|
| 413 |
+
description: comparing a 27-char truncated ERP payee against a string that
|
| 414 |
+
also carries "Payment Instruction", a reference number and a channel
|
| 415 |
+
suffix diluted the ratio far below any usable threshold, so this returned
|
| 416 |
+
False even for obvious pairs and the tolerance pass never fired.
|
| 417 |
+
Containment handles the truncation in either direction.
|
| 418 |
+
"""
|
| 419 |
+
a, b = _norm_name(a), _norm_name(_bank_payee(b))
|
| 420 |
if not a or not b:
|
| 421 |
return False
|
| 422 |
+
if a in b or b in a:
|
| 423 |
return True
|
| 424 |
return difflib.SequenceMatcher(None, a, b).ratio() >= 0.82
|
| 425 |
|
|
|
|
| 612 |
})
|
| 613 |
out = out[out["amount"].round(2) != 0].reset_index(drop=True)
|
| 614 |
out["refs"] = out["desc"].map(lambda s: set(_REF_RE.findall(s)))
|
| 615 |
+
# A reversal carries the same magnitude as the payment it backs out, so it
|
| 616 |
+
# competes for the same match. Flagged here and ranked last, so it is only
|
| 617 |
+
# taken when nothing else fits.
|
| 618 |
+
out["is_reversal"] = out["desc"].str.contains(
|
| 619 |
+
r"revers|returned|nsf|chargeback|stop\s*pay", case=False, regex=True, na=False)
|
| 620 |
|
| 621 |
n_ref = int(out["refs"].map(bool).sum())
|
| 622 |
logger.info(f"π¦ Bank normalised: {len(out)} transaction(s), "
|
|
|
|
| 658 |
matches = []
|
| 659 |
|
| 660 |
def take(ei, bi, method, confidence, erp_rows=None):
|
| 661 |
+
# A roll-up match owns several ERP entries, so the reported amount and
|
| 662 |
+
# line count are the group's. Reporting only the first entry's figures
|
| 663 |
+
# rendered every roll-up as a matched pair whose two sides disagreed.
|
| 664 |
+
rows = list(erp_rows) if erp_rows else [ei]
|
| 665 |
e, b = erp.loc[ei], bank.loc[bi]
|
| 666 |
+
entries = [str(erp.at[i, "entry"]) for i in rows if _text(erp.at[i, "entry"])]
|
| 667 |
matches.append({
|
| 668 |
+
"erp_entry": ", ".join(entries) if entries else "",
|
| 669 |
+
"erp_date": e["date"],
|
| 670 |
+
"erp_payee": e["payee"],
|
| 671 |
+
"erp_amount": float(erp.loc[rows, "amount"].sum()),
|
| 672 |
+
"erp_lines": int(erp.loc[rows, "n_lines"].sum()),
|
| 673 |
"bank_date": b["date"], "bank_desc": b["desc"], "bank_amount": b["amount"],
|
| 674 |
"method": method, "confidence": confidence,
|
| 675 |
+
"erp_rows": rows,
|
| 676 |
})
|
| 677 |
+
for idx in rows:
|
| 678 |
erp_open.discard(idx)
|
| 679 |
bank_open.discard(bi)
|
| 680 |
|
|
|
|
| 698 |
gap = day_gap(ei, bi)
|
| 699 |
return True if gap is None else gap <= days
|
| 700 |
|
| 701 |
+
has_reversal_flag = "is_reversal" in bank.columns
|
| 702 |
+
|
| 703 |
+
def is_reversal(bi):
|
| 704 |
+
return bool(bank.at[bi, "is_reversal"]) if has_reversal_flag else False
|
| 705 |
+
|
| 706 |
def nearest(ei, options):
|
| 707 |
+
"""Of several equal-amount bank lines, the best candidate.
|
| 708 |
+
|
| 709 |
+
Ordinary lines are preferred over reversals before date is considered.
|
| 710 |
+
August 2026 carried 539,249.00 as two debits and one reversal credit;
|
| 711 |
+
ranking on date alone could hand an ERP payment the reversal, which
|
| 712 |
+
reports the reversal as cleared and leaves a real payment outstanding.
|
| 713 |
+
"""
|
| 714 |
if not options:
|
| 715 |
return None
|
| 716 |
+
return min(options, key=lambda bi: (is_reversal(bi),
|
| 717 |
+
day_gap(ei, bi) is None,
|
| 718 |
day_gap(ei, bi) or 0))
|
| 719 |
|
| 720 |
# ββ Pass 1: the bank's own reference to the ERP entry ββββββββββββββββββββ
|
|
|
|
| 1302 |
- ERP transactions (distribution lines rolled up to entries): {stats['erp_count']} (from {stats['erp_lines']} detail lines)
|
| 1303 |
- {stmt_label} transactions: {stats['bank_count']}
|
| 1304 |
- Matched: {stats['matched']} ({stats['erp_cov']:.1f}% of ERP, {stats['bank_cov']:.1f}% of {stmt_label})
|
| 1305 |
+
- Unmatched in ERP: {stats['erp_unmatched']} worth {_fmt_money(stats['erp_unmatched_value'])} gross
|
| 1306 |
+
- Unmatched in {stmt_label}: {stats['bank_unmatched']} worth {_fmt_money(stats['bank_unmatched_value'])} gross
|
| 1307 |
- Match methods used: {stats['methods']}
|
| 1308 |
|
| 1309 |
LARGEST UNMATCHED {stmt_label.upper()} ITEMS
|
|
|
|
| 1386 |
parts.append(f"| ERP coverage | {stats['erp_cov']:.1f}% |")
|
| 1387 |
parts.append(f"| {stmt_label} coverage | {stats['bank_cov']:.1f}% |")
|
| 1388 |
parts.append(f"| Unmatched in ERP | {stats['erp_unmatched']:,} "
|
| 1389 |
+
f"({_fmt_money(stats['erp_unmatched_value'])} gross) |")
|
| 1390 |
parts.append(f"| Unmatched in {stmt_label} | {stats['bank_unmatched']:,} "
|
| 1391 |
+
f"({_fmt_money(stats['bank_unmatched_value'])} gross) |")
|
| 1392 |
parts.append("")
|
| 1393 |
|
| 1394 |
if steps:
|
|
|
|
| 1461 |
"bank_cov": (len(bank) - len(bank_un)) / len(bank) * 100 if len(bank) else 0.0,
|
| 1462 |
"erp_unmatched": len(erp_un),
|
| 1463 |
"bank_unmatched": len(bank_un),
|
| 1464 |
+
# Gross, not a signed net: unmatched reversal credits would otherwise
|
| 1465 |
+
# cancel unmatched debits and understate the work remaining.
|
| 1466 |
+
"erp_unmatched_value": float(erp_un["amount"].abs().sum()) if not erp_un.empty else 0.0,
|
| 1467 |
+
"bank_unmatched_value": float(bank_un["amount"].abs().sum()) if not bank_un.empty else 0.0,
|
| 1468 |
"methods": methods,
|
| 1469 |
}
|
| 1470 |
|