app/generic_parser.py

import re
from typing import List, Dict, Optional

from .schemas import TransactionRow, DocumentProfile, ParserSpec
from .row_detector import detect_row_blocks, TIME_ONLY_RE
from .cleaner import clean_block_lines, clean_description
from .parsers.etop_parser import parse_event_stream_etop

def is_received_line(x: str) -> bool:
    return "received from" in x.lower()

def is_send_line(x: str) -> bool:
    lx = x.lower()
    return "send to" in lx or "send -" in lx

def is_probable_name_line(x: str) -> bool:
    import re

    x = x.strip()
    if not x:
        return False

    # too long → not a name
    if len(x) > 40:
        return False

    # contains numbers → not a name
    if re.search(r"\d", x):
        return False

    words = x.split()

    # names are usually 1–4 words
    if not (1 <= len(words) <= 4):
        return False

    return all(re.match(r"^[A-Za-z&.\-()]+$", w) for w in words)

def is_system_trace_line(x: str) -> bool:
    import re
    x = x.strip()
    return bool(
        re.fullmatch(r"\d{8}[A-Z]\d{11,}", x) or
        re.fullmatch(r"\d{7,}", x) or
        re.fullmatch(r"page\s+\d+", x.lower())
    )

MONEY_RE = re.compile(r"(?<!\d)(\d{1,3}(?:,\d{3})*(?:\.\d{2})|\d+\.\d{2})(?!\d)")
DATE_RE_DMY_TEXT = re.compile(r"^\d{2}-[A-Za-z]{3}-\d{4}")
DATE_RE_ISO_DT = re.compile(r"^\d{4}-\d{2}-\d{2}T\d{2}:")


def normalize_spaces(text: str) -> str:
    return " ".join(text.split())


def parse_amount(value: str) -> Optional[float]:
    try:
        return float(value.replace(",", ""))
    except Exception:
        return None


def extract_money_values(text: str) -> List[float]:
    values: List[float] = []

    for match in MONEY_RE.findall(text):
        value = parse_amount(match)
        if value is not None:
            values.append(value)

    return values


def is_table_row_start(line: str) -> bool:
    return bool(DATE_RE_DMY_TEXT.match(line.strip()))


def is_event_block_start(line: str) -> bool:
    return bool(DATE_RE_ISO_DT.match(line.strip()))


def dedupe_rows(rows: List[TransactionRow]) -> List[TransactionRow]:
    seen = set()
    result: List[TransactionRow] = []

    for row in rows:
        key = (
            row.date,
            row.description,
            row.debit,
            row.credit,
            row.balance,
            row.source_page,
        )
        if key in seen:
            continue
        seen.add(key)
        result.append(row)

    return result


def parse_event_blocks_from_detected(blocks: List[Dict]) -> List[TransactionRow]:
    rows: List[TransactionRow] = []

    for block in blocks:
        page_no = block["page"]
        lines = clean_block_lines(block["lines"])

        if not lines:
            continue

        first_line = lines[0]
        if "T" not in first_line:
            continue

        current_date = first_line.split("T")[0]

        meaningful = [line for line in lines[1:] if not TIME_ONLY_RE.match(line)]
        if not meaningful:
            continue

        amount_line = None
        for candidate in reversed(meaningful):
            if len(extract_money_values(candidate)) >= 2:
                amount_line = candidate
                break

        if not amount_line:
            continue

        money_vals = extract_money_values(amount_line)
        if len(money_vals) < 2:
            continue

        debit = None
        credit = None
        balance = None

        if len(money_vals) >= 3:
            debit = money_vals[-3]
            credit = money_vals[-2]
            balance = money_vals[-1]
        else:
            tx_amount = money_vals[-2]
            balance = money_vals[-1]
            lower_text = " ".join(meaningful).lower()

            if "credit" in lower_text or "from " in lower_text or "reversal" in lower_text:
                credit = tx_amount
            else:
                debit = tx_amount

        description_parts: List[str] = []
        for line in meaningful:
            if line == amount_line:
                stripped = line
                for money in MONEY_RE.findall(line):
                    stripped = stripped.replace(money, " ")
                description_parts.append(stripped)
            else:
                description_parts.append(line)

        description = clean_description(" ".join(description_parts))
        if not description:
            continue

        rows.append(
            TransactionRow(
                date=current_date,
                description=description,
                debit=debit if debit not in (0, 0.0) else None,
                credit=credit if credit not in (0, 0.0) else None,
                balance=balance,
                source_page=page_no,
                confidence="medium",
            )
        )

    return rows


def parse_table_blocks_from_detected(blocks: List[Dict], parser_spec: ParserSpec) -> List[TransactionRow]:
    rows: List[TransactionRow] = []

    debit_keywords = [
        "outward",
        "transfer to",
        "charges",
        "charge",
        "vat",
        "levy",
        "purchase",
        "payment",
        "airtime",
        "pos",
        "nibss instant payment outward",
        "commission on nip transfer",
        "electronic money transfer levy",
        "debit",
        "withdrawal",
        "send to",
        "auto deduct",
        "cashbox auto save",
        "betting deposit",
        "buy data bundle",
        "top up airtime",
        "card payment",
        "spend & save deposit",
    ]

    credit_keywords = [
        "inward",
        "transfer from",
        "from ",
        "reversed",
        "reversal",
        "credit",
        "deposit",
        "transfer between customers",
        "own account transfer",
        "refund",
        "received from",
        "merchant disbursement",
        "cashbox withdraw",
        "disbursement-cash loan",
        "free travel share cash",
        "cashbox interest",
        "spend & save interest",
        "cashbox interest",
    ]

    for block in blocks:
        page_no = block["page"]
        lines = clean_block_lines(block["lines"])

        if not lines:
            continue

        first_line = lines[0]
        parts = first_line.split()
        if not parts:
            continue

        transaction_date = None

        if parser_spec.row_start_pattern == "line_starts_with_two_dates":
            if len(parts) < 2:
                continue
            transaction_date = parts[0]
        elif parser_spec.row_start_pattern == "line_starts_with_slash_date":
            transaction_date = parts[0]
        else:
            transaction_date = parts[0]

        block_text = " ".join(lines)
        lower_text = block_text.lower()

        money_vals = extract_money_values(block_text)

        if len(money_vals) < 2:
            continue

        debit = None
        credit = None
        balance = None

        if parser_spec.amount_style == "debit_credit_balance":
            if len(money_vals) == 2:
                tx_amount = money_vals[0]
                balance = money_vals[1]

                if any(keyword in lower_text for keyword in credit_keywords):
                    credit = tx_amount
                else:
                    debit = tx_amount

            elif len(money_vals) >= 3:
                last_three = money_vals[-3:]

                tx_amount = money_vals[-2]
                balance = money_vals[-1]

                if any(keyword in lower_text for keyword in credit_keywords):
                    credit = tx_amount
                elif any(keyword in lower_text for keyword in debit_keywords):
                    debit = tx_amount
                else:
                    debit = tx_amount

                if (
                    " debits " in f" {lower_text} "
                    or " credits " in f" {lower_text} "
                ) and len(last_three) == 3:
                    possible_debit = last_three[0]
                    possible_credit = last_three[1]
                    possible_balance = last_three[2]

                    if possible_debit == 0 and possible_credit != 0:
                        debit = None
                        credit = possible_credit
                        balance = possible_balance
                    elif possible_credit == 0 and possible_debit != 0:
                        debit = possible_debit
                        credit = None
                        balance = possible_balance

        elif parser_spec.amount_style == "amount_balance":
            tx_amount = money_vals[-2]
            balance = money_vals[-1]

            if parser_spec.direction_signal == "keywords":
                if any(keyword in lower_text for keyword in credit_keywords):
                    credit = tx_amount
                else:
                    debit = tx_amount
            else:
                debit = tx_amount

        elif parser_spec.amount_style == "signed_amount":
            signed_match = re.search(r"([+-]\d+\.\d{2})", block_text)
            if signed_match:
                amount = float(signed_match.group())
                if amount < 0:
                    debit = abs(amount)
                else:
                    credit = amount
            balance = None

        else:
            tx_amount = money_vals[-2]
            balance = money_vals[-1]

            if any(keyword in lower_text for keyword in credit_keywords):
                credit = tx_amount
            else:
                debit = tx_amount

        description = block_text

        if parser_spec.row_start_pattern == "line_starts_with_two_dates":
            description = re.sub(
                r"^\d{2}-[A-Za-z]{3}-\d{4}\s+\d{2}-[A-Za-z]{3}-\d{4}",
                "",
                description,
                count=1,
            )
        elif parser_spec.row_start_pattern == "line_starts_with_slash_date":
            description = re.sub(
                r"^\d{2}/\d{2}/\d{2}",
                "",
                description,
                count=1,
            )

        all_money = MONEY_RE.findall(description)

        if parser_spec.amount_style in ("debit_credit_balance", "amount_balance") and len(all_money) >= 2:
            for money in all_money[-2:]:
                description = description[::-1].replace(money[::-1], " ", 1)[::-1]

        description = clean_description(description)

        if not description:
            continue

        rows.append(
            TransactionRow(
                date=transaction_date,
                description=description,
                debit=debit if debit not in (0, 0.0) else None,
                credit=credit if credit not in (0, 0.0) else None,
                balance=balance,
                source_page=page_no,
                confidence="medium",
            )
        )

    return rows


def parse_event_stream(page_texts, parser_spec):
    import re

    rows = []

    DATE_RE = re.compile(r"\d{2}/\d{2}/\d{2}")
    AMOUNT_RE = re.compile(r"\d{1,3}(?:,\d{3})*\.\d{2}")

    for page in page_texts:
        lines = [l.strip() for l in page["text"].splitlines() if l.strip()]

        current = {
            "date": None,
            "desc": [],
            "debit": None,
            "credit": None,
            "balance": None
        }

        for line in lines:

            # 🔹 STEP 1: detect new transaction
            if DATE_RE.search(line):

                # save previous
                if current["date"] and (current["debit"] or current["credit"]):
                    rows.append(TransactionRow(
                        date=current["date"],
                        description=clean_description(" ".join(current["desc"])),
                        debit=current["debit"],
                        credit=current["credit"],
                        balance=current["balance"],
                        source_page=page["page"],
                        confidence="medium"
                    ))

                # start new transaction
                current = {
                    "date": DATE_RE.search(line).group(),
                    "desc": [line],
                    "debit": None,
                    "credit": None,
                    "balance": None
                }

                continue

            # 🔹 STEP 2: detect amount line
            amounts = AMOUNT_RE.findall(line)

            if len(amounts) >= 2:
                values = [float(a.replace(",", "")) for a in amounts]

                amount = values[0]
                balance = values[-1]

                current["balance"] = balance

                lower = " ".join(current["desc"]).lower()

                if "inward" in lower or "from" in lower:
                    current["credit"] = amount
                else:
                    current["debit"] = amount

                continue

            # 🔹 STEP 3: description continuation
            current["desc"].append(line)

        # save last row
        if current["date"] and (current["debit"] or current["credit"]):
            rows.append(TransactionRow(
                date=current["date"],
                description=clean_description(" ".join(current["desc"])),
                debit=current["debit"],
                credit=current["credit"],
                balance=current["balance"],
                source_page=page["page"],
                confidence="medium"
            ))

    return rows

def build_stream_row(lines, page_no):
    import re

    date = None
    debit = None
    credit = None
    balance = None

    description_parts = []

    for line in lines:
        # extract date
        if not date:
            date_match = re.search(r"\d{2}/\d{2}/\d{2}", line)
            if date_match:
                date = date_match.group()

        # extract amounts
        amounts = re.findall(r"\d{1,3}(?:,\d{3})*\.\d{2}", line)
        values = [float(a.replace(",", "")) for a in amounts]

        if len(values) >= 2:
            # likely amount + balance
            amount = values[0]
            balance = values[-1]

            lower = line.lower()

            if "inward" in lower or "from" in lower or "deposit" in lower:
                credit = amount
            else:
                debit = amount

        # collect description
        description_parts.append(line)

    if not date or (debit is None and credit is None):
        return None

    description = " ".join(description_parts)

    return TransactionRow(
        date=date,
        description=clean_description(description),
        debit=debit,
        credit=credit,
        balance=balance,
        source_page=page_no,
        confidence="medium",
    )

def parse_with_profile(
    page_texts: List[Dict],
    profile: DocumentProfile,
    parser_spec: ParserSpec,
) -> List[TransactionRow]:
    if parser_spec.layout_type == "event_stream_hybrid":
        rows = parse_event_stream_hybrid(page_texts)
        return dedupe_rows(rows)

    if parser_spec.layout_type == "event_stream_multiline":
        rows = parse_event_stream(page_texts, parser_spec)
        return dedupe_rows(rows)

    if parser_spec.layout_type == "event_stream_singleline":
        rows = parse_event_stream_singleline(page_texts, parser_spec)
        return dedupe_rows(rows)

    blocks = detect_row_blocks(page_texts, parser_spec=parser_spec)

    if parser_spec.layout_type == "event_block":
        rows = parse_event_blocks_from_detected(blocks)
        return dedupe_rows(rows)

    rows = parse_table_blocks_from_detected(blocks, parser_spec)
    return dedupe_rows(rows)

    
def is_summary_or_header(line: str) -> bool:
    lower = line.lower().strip()

    summary_keywords = [
        "opening balance",
        "closing balance",
        "summary",
        "money in",
        "money out",
        "statement period",
        "date/time",
        "transaction date",
        "transaction detail",
        "transaction id",
    ]

    # exact / strong matches only
    return any(keyword in lower for keyword in summary_keywords)

def looks_like_summary_block(text: str) -> bool:
    lower = text.lower()

    strong_markers = [
        "opening balance",
        "closing balance",
        "summary",
        "money in",
        "money out",
    ]

    hits = sum(1 for marker in strong_markers if marker in lower)
    return hits >= 2

def parse_event_stream_singleline(page_texts, parser_spec):
    import re

    rows = []

    DATE_RE = re.compile(r"\d{2}/\d{2}/\d{4}")
    AMOUNT_RE = re.compile(r"[+-]\d+\.\d{2}")

    for page in page_texts:
        lines = [l.strip() for l in page["text"].splitlines() if l.strip()]

        for line in lines:

            if not DATE_RE.search(line):
                continue

            amount_match = AMOUNT_RE.search(line)

            if not amount_match:
                continue

            amount = float(amount_match.group().replace("+", "").replace(",", ""))

            if amount > 0:
                credit = amount
                debit = None
            else:
                debit = abs(amount)
                credit = None

            rows.append(TransactionRow(
                date=DATE_RE.search(line).group(),
                description=clean_description(line),
                debit=debit,
                credit=credit,
                balance=None,
                source_page=page["page"],
                confidence="high"
            ))

    return rows


def parse_event_stream_hybrid(page_texts):
    import re

    rows = []

    DATETIME_RE = re.compile(
        r"^(\d{2}/\d{2}/\d{4})\s+(\d{2}:\d{2}:\d{2}\s+(?:AM|PM))\b"
    )
    AMOUNT_RE = re.compile(r"([+-])(\d{1,3}(?:,\d{3})*\.\d{2})")
    TXN_ID_RE = re.compile(r"\b([A-Za-z0-9_]{10,})\b")
    PURE_NUMERIC_REF_RE = re.compile(r"^\d{7,}$")
    PAGE_NUM_RE = re.compile(r"^\d+$")
    PALMPAY_TRACE_RE = re.compile(r"^\d{8}[A-Z]\d{11,}$")
    LONG_TRACE_RE = re.compile(r"\b20\d{14,}\b")

    def clean_text(s: str) -> str:
        s = s.replace("\uFFFE", " ")
        s = " ".join(s.split())
        return s.strip()

    def strip_datetime_prefix(s: str) -> str:
        return re.sub(
            r"^\d{2}/\d{2}/\d{4}\s+\d{2}:\d{2}:\d{2}\s+(?:AM|PM)\s*",
            "",
            s,
        ).strip()

    def strip_amount(s: str) -> str:
        return re.sub(r"([+-])(\d{1,3}(?:,\d{3})*\.\d{2})", "", s).strip()

    def strip_txn_id_text(s: str) -> str:
        s = TXN_ID_RE.sub("", s)
        s = LONG_TRACE_RE.sub("", s)
        return clean_text(s)

    def is_noise_line(x: str) -> bool:
        if not x:
            return True
        if PAGE_NUM_RE.match(x):
            return True
        if PURE_NUMERIC_REF_RE.match(x):
            return True
        if PALMPAY_TRACE_RE.match(x):
            return True
        return False

    def extract_txn_id(parts):
        for p in reversed(parts):
            m = TXN_ID_RE.search(p)
            if m:
                return m.group(1)
        return None

    def build_row(date, page_no, pre_lines, txn_line, post_lines):
        line_body = strip_datetime_prefix(clean_text(txn_line))
        amt_match = AMOUNT_RE.search(line_body)
        if not amt_match:
            return None

        sign = amt_match.group(1)
        amount = float(amt_match.group(2).replace(",", ""))

        debit = amount if sign == "-" else None
        credit = amount if sign == "+" else None
        is_credit = credit is not None
        is_debit = debit is not None

        txn_desc = strip_amount(line_body)
        txn_desc = strip_txn_id_text(txn_desc)

        pre = [clean_text(x) for x in pre_lines if clean_text(x)]
        post = [clean_text(x) for x in post_lines if clean_text(x)]

        pre = [x for x in pre if not is_noise_line(x)]
        post = [x for x in post if not is_noise_line(x)]

        txn_id = extract_txn_id([txn_desc] + pre + post)

        cleaned_pre = []
        for x in pre:
            if txn_id:
                x = x.replace(txn_id, "")
            x = strip_txn_id_text(x)
            if x and not is_noise_line(x):
                cleaned_pre.append(x)

        cleaned_post = []
        for x in post:
            if txn_id:
                x = x.replace(txn_id, "")
            x = strip_txn_id_text(x)
            if x and not is_noise_line(x):
                cleaned_post.append(x)

        if txn_id:
            txn_desc = txn_desc.replace(txn_id, "")
            txn_desc = clean_text(txn_desc)

        desc_parts = []

        # PRE LINES
        for x in cleaned_pre:
            if is_credit:
                if is_received_line(x):
                    desc_parts.append(x)
                elif not is_send_line(x) and not is_probable_name_line(x):
                    desc_parts.append(x)
            elif is_debit:
                if is_send_line(x):
                    desc_parts.append(x)
                elif not is_received_line(x) and not is_probable_name_line(x):
                    desc_parts.append(x)

        # MAIN DESCRIPTION
        if txn_desc:
            desc_parts.append(txn_desc)

        # POST LINES
        for x in cleaned_post:
            if is_credit:
                if is_probable_name_line(x):
                    desc_parts.append(x)
                elif not is_send_line(x):
                    # allow helpful continuation like "KORAPAY", "FINANCE LIMITED"
                    if not is_received_line(x):
                        desc_parts.append(x)
            elif is_debit:
                if is_probable_name_line(x):
                    # only keep name fragments for debit if a send-line exists
                    if any(is_send_line(p) for p in cleaned_pre):
                        desc_parts.append(x)
                elif not is_received_line(x):
                    # allow continuation like FINANCE LTD), LIMITED, etc.
                    if not is_send_line(x):
                        desc_parts.append(x)

        description = clean_text(" ".join(desc_parts))
        description = LONG_TRACE_RE.sub("", description)
        description = clean_text(description)

        # final safety cleanup
        description = re.sub(r"\b[A-Za-z0-9_]{10,}\b", "", description)
        description = clean_text(description)

        if not description:
            description = txn_desc

        if not description:
            return None

        return TransactionRow(
            date=date,
            description=description,
            debit=debit,
            credit=credit,
            balance=None,
            source_page=page_no,
            confidence="high",
        )

    for page in page_texts:
        raw_lines = [line.strip() for line in page["text"].splitlines() if line.strip()]
        lines = [clean_text(x) for x in raw_lines if clean_text(x)]

        pending_lines = []
        i = 0

        while i < len(lines):
            line = lines[i]
            dt_match = DATETIME_RE.match(line)

            if not dt_match:
                pending_lines.append(line)
                i += 1
                continue

            date = dt_match.group(1)
            txn_line = line

            post_lines = []
            j = i + 1
            while j < len(lines) and not DATETIME_RE.match(lines[j]):
                post_lines.append(lines[j])
                j += 1

            row = build_row(
                date=date,
                page_no=page["page"],
                pre_lines=pending_lines,
                txn_line=txn_line,
                post_lines=post_lines,
            )

            if row:
                rows.append(row)

            pending_lines = []
            i = j

    return rows