import re from typing import List, Dict, Optional from .schemas import TransactionRow, DocumentProfile, ParserSpec from .row_detector import detect_row_blocks, TIME_ONLY_RE from .cleaner import clean_block_lines, clean_description from .parsers.etop_parser import parse_event_stream_etop def is_received_line(x: str) -> bool: return "received from" in x.lower() def is_send_line(x: str) -> bool: lx = x.lower() return "send to" in lx or "send -" in lx def is_probable_name_line(x: str) -> bool: import re x = x.strip() if not x: return False # too long → not a name if len(x) > 40: return False # contains numbers → not a name if re.search(r"\d", x): return False words = x.split() # names are usually 1–4 words if not (1 <= len(words) <= 4): return False return all(re.match(r"^[A-Za-z&.\-()]+$", w) for w in words) def is_system_trace_line(x: str) -> bool: import re x = x.strip() return bool( re.fullmatch(r"\d{8}[A-Z]\d{11,}", x) or re.fullmatch(r"\d{7,}", x) or re.fullmatch(r"page\s+\d+", x.lower()) ) MONEY_RE = re.compile(r"(? str: return " ".join(text.split()) def parse_amount(value: str) -> Optional[float]: try: return float(value.replace(",", "")) except Exception: return None def extract_money_values(text: str) -> List[float]: values: List[float] = [] for match in MONEY_RE.findall(text): value = parse_amount(match) if value is not None: values.append(value) return values def is_table_row_start(line: str) -> bool: return bool(DATE_RE_DMY_TEXT.match(line.strip())) def is_event_block_start(line: str) -> bool: return bool(DATE_RE_ISO_DT.match(line.strip())) def dedupe_rows(rows: List[TransactionRow]) -> List[TransactionRow]: seen = set() result: List[TransactionRow] = [] for row in rows: key = ( row.date, row.description, row.debit, row.credit, row.balance, row.source_page, ) if key in seen: continue seen.add(key) result.append(row) return result def parse_event_blocks_from_detected(blocks: List[Dict]) -> List[TransactionRow]: rows: List[TransactionRow] = [] for block in blocks: page_no = block["page"] lines = clean_block_lines(block["lines"]) if not lines: continue first_line = lines[0] if "T" not in first_line: continue current_date = first_line.split("T")[0] meaningful = [line for line in lines[1:] if not TIME_ONLY_RE.match(line)] if not meaningful: continue amount_line = None for candidate in reversed(meaningful): if len(extract_money_values(candidate)) >= 2: amount_line = candidate break if not amount_line: continue money_vals = extract_money_values(amount_line) if len(money_vals) < 2: continue debit = None credit = None balance = None if len(money_vals) >= 3: debit = money_vals[-3] credit = money_vals[-2] balance = money_vals[-1] else: tx_amount = money_vals[-2] balance = money_vals[-1] lower_text = " ".join(meaningful).lower() if "credit" in lower_text or "from " in lower_text or "reversal" in lower_text: credit = tx_amount else: debit = tx_amount description_parts: List[str] = [] for line in meaningful: if line == amount_line: stripped = line for money in MONEY_RE.findall(line): stripped = stripped.replace(money, " ") description_parts.append(stripped) else: description_parts.append(line) description = clean_description(" ".join(description_parts)) if not description: continue rows.append( TransactionRow( date=current_date, description=description, debit=debit if debit not in (0, 0.0) else None, credit=credit if credit not in (0, 0.0) else None, balance=balance, source_page=page_no, confidence="medium", ) ) return rows def parse_table_blocks_from_detected(blocks: List[Dict], parser_spec: ParserSpec) -> List[TransactionRow]: rows: List[TransactionRow] = [] debit_keywords = [ "outward", "transfer to", "charges", "charge", "vat", "levy", "purchase", "payment", "airtime", "pos", "nibss instant payment outward", "commission on nip transfer", "electronic money transfer levy", "debit", "withdrawal", "send to", "auto deduct", "cashbox auto save", "betting deposit", "buy data bundle", "top up airtime", "card payment", "spend & save deposit", ] credit_keywords = [ "inward", "transfer from", "from ", "reversed", "reversal", "credit", "deposit", "transfer between customers", "own account transfer", "refund", "received from", "merchant disbursement", "cashbox withdraw", "disbursement-cash loan", "free travel share cash", "cashbox interest", "spend & save interest", "cashbox interest", ] for block in blocks: page_no = block["page"] lines = clean_block_lines(block["lines"]) if not lines: continue first_line = lines[0] parts = first_line.split() if not parts: continue transaction_date = None if parser_spec.row_start_pattern == "line_starts_with_two_dates": if len(parts) < 2: continue transaction_date = parts[0] elif parser_spec.row_start_pattern == "line_starts_with_slash_date": transaction_date = parts[0] else: transaction_date = parts[0] block_text = " ".join(lines) lower_text = block_text.lower() money_vals = extract_money_values(block_text) if len(money_vals) < 2: continue debit = None credit = None balance = None if parser_spec.amount_style == "debit_credit_balance": if len(money_vals) == 2: tx_amount = money_vals[0] balance = money_vals[1] if any(keyword in lower_text for keyword in credit_keywords): credit = tx_amount else: debit = tx_amount elif len(money_vals) >= 3: last_three = money_vals[-3:] tx_amount = money_vals[-2] balance = money_vals[-1] if any(keyword in lower_text for keyword in credit_keywords): credit = tx_amount elif any(keyword in lower_text for keyword in debit_keywords): debit = tx_amount else: debit = tx_amount if ( " debits " in f" {lower_text} " or " credits " in f" {lower_text} " ) and len(last_three) == 3: possible_debit = last_three[0] possible_credit = last_three[1] possible_balance = last_three[2] if possible_debit == 0 and possible_credit != 0: debit = None credit = possible_credit balance = possible_balance elif possible_credit == 0 and possible_debit != 0: debit = possible_debit credit = None balance = possible_balance elif parser_spec.amount_style == "amount_balance": tx_amount = money_vals[-2] balance = money_vals[-1] if parser_spec.direction_signal == "keywords": if any(keyword in lower_text for keyword in credit_keywords): credit = tx_amount else: debit = tx_amount else: debit = tx_amount elif parser_spec.amount_style == "signed_amount": signed_match = re.search(r"([+-]\d+\.\d{2})", block_text) if signed_match: amount = float(signed_match.group()) if amount < 0: debit = abs(amount) else: credit = amount balance = None else: tx_amount = money_vals[-2] balance = money_vals[-1] if any(keyword in lower_text for keyword in credit_keywords): credit = tx_amount else: debit = tx_amount description = block_text if parser_spec.row_start_pattern == "line_starts_with_two_dates": description = re.sub( r"^\d{2}-[A-Za-z]{3}-\d{4}\s+\d{2}-[A-Za-z]{3}-\d{4}", "", description, count=1, ) elif parser_spec.row_start_pattern == "line_starts_with_slash_date": description = re.sub( r"^\d{2}/\d{2}/\d{2}", "", description, count=1, ) all_money = MONEY_RE.findall(description) if parser_spec.amount_style in ("debit_credit_balance", "amount_balance") and len(all_money) >= 2: for money in all_money[-2:]: description = description[::-1].replace(money[::-1], " ", 1)[::-1] description = clean_description(description) if not description: continue rows.append( TransactionRow( date=transaction_date, description=description, debit=debit if debit not in (0, 0.0) else None, credit=credit if credit not in (0, 0.0) else None, balance=balance, source_page=page_no, confidence="medium", ) ) return rows def parse_event_stream(page_texts, parser_spec): import re rows = [] DATE_RE = re.compile(r"\d{2}/\d{2}/\d{2}") AMOUNT_RE = re.compile(r"\d{1,3}(?:,\d{3})*\.\d{2}") for page in page_texts: lines = [l.strip() for l in page["text"].splitlines() if l.strip()] current = { "date": None, "desc": [], "debit": None, "credit": None, "balance": None } for line in lines: # 🔹 STEP 1: detect new transaction if DATE_RE.search(line): # save previous if current["date"] and (current["debit"] or current["credit"]): rows.append(TransactionRow( date=current["date"], description=clean_description(" ".join(current["desc"])), debit=current["debit"], credit=current["credit"], balance=current["balance"], source_page=page["page"], confidence="medium" )) # start new transaction current = { "date": DATE_RE.search(line).group(), "desc": [line], "debit": None, "credit": None, "balance": None } continue # 🔹 STEP 2: detect amount line amounts = AMOUNT_RE.findall(line) if len(amounts) >= 2: values = [float(a.replace(",", "")) for a in amounts] amount = values[0] balance = values[-1] current["balance"] = balance lower = " ".join(current["desc"]).lower() if "inward" in lower or "from" in lower: current["credit"] = amount else: current["debit"] = amount continue # 🔹 STEP 3: description continuation current["desc"].append(line) # save last row if current["date"] and (current["debit"] or current["credit"]): rows.append(TransactionRow( date=current["date"], description=clean_description(" ".join(current["desc"])), debit=current["debit"], credit=current["credit"], balance=current["balance"], source_page=page["page"], confidence="medium" )) return rows def build_stream_row(lines, page_no): import re date = None debit = None credit = None balance = None description_parts = [] for line in lines: # extract date if not date: date_match = re.search(r"\d{2}/\d{2}/\d{2}", line) if date_match: date = date_match.group() # extract amounts amounts = re.findall(r"\d{1,3}(?:,\d{3})*\.\d{2}", line) values = [float(a.replace(",", "")) for a in amounts] if len(values) >= 2: # likely amount + balance amount = values[0] balance = values[-1] lower = line.lower() if "inward" in lower or "from" in lower or "deposit" in lower: credit = amount else: debit = amount # collect description description_parts.append(line) if not date or (debit is None and credit is None): return None description = " ".join(description_parts) return TransactionRow( date=date, description=clean_description(description), debit=debit, credit=credit, balance=balance, source_page=page_no, confidence="medium", ) def parse_with_profile( page_texts: List[Dict], profile: DocumentProfile, parser_spec: ParserSpec, ) -> List[TransactionRow]: if parser_spec.layout_type == "event_stream_hybrid": rows = parse_event_stream_hybrid(page_texts) return dedupe_rows(rows) if parser_spec.layout_type == "event_stream_multiline": rows = parse_event_stream(page_texts, parser_spec) return dedupe_rows(rows) if parser_spec.layout_type == "event_stream_singleline": rows = parse_event_stream_singleline(page_texts, parser_spec) return dedupe_rows(rows) blocks = detect_row_blocks(page_texts, parser_spec=parser_spec) if parser_spec.layout_type == "event_block": rows = parse_event_blocks_from_detected(blocks) return dedupe_rows(rows) rows = parse_table_blocks_from_detected(blocks, parser_spec) return dedupe_rows(rows) def is_summary_or_header(line: str) -> bool: lower = line.lower().strip() summary_keywords = [ "opening balance", "closing balance", "summary", "money in", "money out", "statement period", "date/time", "transaction date", "transaction detail", "transaction id", ] # exact / strong matches only return any(keyword in lower for keyword in summary_keywords) def looks_like_summary_block(text: str) -> bool: lower = text.lower() strong_markers = [ "opening balance", "closing balance", "summary", "money in", "money out", ] hits = sum(1 for marker in strong_markers if marker in lower) return hits >= 2 def parse_event_stream_singleline(page_texts, parser_spec): import re rows = [] DATE_RE = re.compile(r"\d{2}/\d{2}/\d{4}") AMOUNT_RE = re.compile(r"[+-]\d+\.\d{2}") for page in page_texts: lines = [l.strip() for l in page["text"].splitlines() if l.strip()] for line in lines: if not DATE_RE.search(line): continue amount_match = AMOUNT_RE.search(line) if not amount_match: continue amount = float(amount_match.group().replace("+", "").replace(",", "")) if amount > 0: credit = amount debit = None else: debit = abs(amount) credit = None rows.append(TransactionRow( date=DATE_RE.search(line).group(), description=clean_description(line), debit=debit, credit=credit, balance=None, source_page=page["page"], confidence="high" )) return rows def parse_event_stream_hybrid(page_texts): import re rows = [] DATETIME_RE = re.compile( r"^(\d{2}/\d{2}/\d{4})\s+(\d{2}:\d{2}:\d{2}\s+(?:AM|PM))\b" ) AMOUNT_RE = re.compile(r"([+-])(\d{1,3}(?:,\d{3})*\.\d{2})") TXN_ID_RE = re.compile(r"\b([A-Za-z0-9_]{10,})\b") PURE_NUMERIC_REF_RE = re.compile(r"^\d{7,}$") PAGE_NUM_RE = re.compile(r"^\d+$") PALMPAY_TRACE_RE = re.compile(r"^\d{8}[A-Z]\d{11,}$") LONG_TRACE_RE = re.compile(r"\b20\d{14,}\b") def clean_text(s: str) -> str: s = s.replace("\uFFFE", " ") s = " ".join(s.split()) return s.strip() def strip_datetime_prefix(s: str) -> str: return re.sub( r"^\d{2}/\d{2}/\d{4}\s+\d{2}:\d{2}:\d{2}\s+(?:AM|PM)\s*", "", s, ).strip() def strip_amount(s: str) -> str: return re.sub(r"([+-])(\d{1,3}(?:,\d{3})*\.\d{2})", "", s).strip() def strip_txn_id_text(s: str) -> str: s = TXN_ID_RE.sub("", s) s = LONG_TRACE_RE.sub("", s) return clean_text(s) def is_noise_line(x: str) -> bool: if not x: return True if PAGE_NUM_RE.match(x): return True if PURE_NUMERIC_REF_RE.match(x): return True if PALMPAY_TRACE_RE.match(x): return True return False def extract_txn_id(parts): for p in reversed(parts): m = TXN_ID_RE.search(p) if m: return m.group(1) return None def build_row(date, page_no, pre_lines, txn_line, post_lines): line_body = strip_datetime_prefix(clean_text(txn_line)) amt_match = AMOUNT_RE.search(line_body) if not amt_match: return None sign = amt_match.group(1) amount = float(amt_match.group(2).replace(",", "")) debit = amount if sign == "-" else None credit = amount if sign == "+" else None is_credit = credit is not None is_debit = debit is not None txn_desc = strip_amount(line_body) txn_desc = strip_txn_id_text(txn_desc) pre = [clean_text(x) for x in pre_lines if clean_text(x)] post = [clean_text(x) for x in post_lines if clean_text(x)] pre = [x for x in pre if not is_noise_line(x)] post = [x for x in post if not is_noise_line(x)] txn_id = extract_txn_id([txn_desc] + pre + post) cleaned_pre = [] for x in pre: if txn_id: x = x.replace(txn_id, "") x = strip_txn_id_text(x) if x and not is_noise_line(x): cleaned_pre.append(x) cleaned_post = [] for x in post: if txn_id: x = x.replace(txn_id, "") x = strip_txn_id_text(x) if x and not is_noise_line(x): cleaned_post.append(x) if txn_id: txn_desc = txn_desc.replace(txn_id, "") txn_desc = clean_text(txn_desc) desc_parts = [] # PRE LINES for x in cleaned_pre: if is_credit: if is_received_line(x): desc_parts.append(x) elif not is_send_line(x) and not is_probable_name_line(x): desc_parts.append(x) elif is_debit: if is_send_line(x): desc_parts.append(x) elif not is_received_line(x) and not is_probable_name_line(x): desc_parts.append(x) # MAIN DESCRIPTION if txn_desc: desc_parts.append(txn_desc) # POST LINES for x in cleaned_post: if is_credit: if is_probable_name_line(x): desc_parts.append(x) elif not is_send_line(x): # allow helpful continuation like "KORAPAY", "FINANCE LIMITED" if not is_received_line(x): desc_parts.append(x) elif is_debit: if is_probable_name_line(x): # only keep name fragments for debit if a send-line exists if any(is_send_line(p) for p in cleaned_pre): desc_parts.append(x) elif not is_received_line(x): # allow continuation like FINANCE LTD), LIMITED, etc. if not is_send_line(x): desc_parts.append(x) description = clean_text(" ".join(desc_parts)) description = LONG_TRACE_RE.sub("", description) description = clean_text(description) # final safety cleanup description = re.sub(r"\b[A-Za-z0-9_]{10,}\b", "", description) description = clean_text(description) if not description: description = txn_desc if not description: return None return TransactionRow( date=date, description=description, debit=debit, credit=credit, balance=None, source_page=page_no, confidence="high", ) for page in page_texts: raw_lines = [line.strip() for line in page["text"].splitlines() if line.strip()] lines = [clean_text(x) for x in raw_lines if clean_text(x)] pending_lines = [] i = 0 while i < len(lines): line = lines[i] dt_match = DATETIME_RE.match(line) if not dt_match: pending_lines.append(line) i += 1 continue date = dt_match.group(1) txn_line = line post_lines = [] j = i + 1 while j < len(lines) and not DATETIME_RE.match(lines[j]): post_lines.append(lines[j]) j += 1 row = build_row( date=date, page_no=page["page"], pre_lines=pending_lines, txn_line=txn_line, post_lines=post_lines, ) if row: rows.append(row) pending_lines = [] i = j return rows