Spaces:
Sleeping
Sleeping
Upload folder using huggingface_hub
Browse files- statement_parser.py +52 -36
statement_parser.py
CHANGED
|
@@ -671,16 +671,32 @@ def get_multi_page_sample(lines_input, lines_per_page=35):
|
|
| 671 |
|
| 672 |
return "\n".join(sample)
|
| 673 |
|
| 674 |
-
def extract_direct_transactions_from_llm(
|
| 675 |
"""
|
| 676 |
Direct LLM transaction extractor as a fail-safe fallback.
|
| 677 |
-
|
| 678 |
"""
|
| 679 |
-
|
| 680 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 681 |
|
| 682 |
--- STATEMENT LINES ---
|
| 683 |
-
{
|
| 684 |
--- END STATEMENT LINES ---
|
| 685 |
|
| 686 |
Return ONLY a valid JSON array of objects with keys: "date", "description", "debit", "credit", "balance".
|
|
@@ -691,35 +707,34 @@ Output ONLY inside a ```json ``` code block:
|
|
| 691 |
]
|
| 692 |
```
|
| 693 |
"""
|
| 694 |
-
|
| 695 |
-
|
| 696 |
-
|
| 697 |
-
|
| 698 |
-
|
| 699 |
-
|
| 700 |
-
|
| 701 |
-
|
| 702 |
-
|
| 703 |
-
|
| 704 |
-
|
| 705 |
-
|
| 706 |
-
|
| 707 |
-
|
| 708 |
-
|
| 709 |
-
|
| 710 |
-
|
| 711 |
-
|
| 712 |
-
|
| 713 |
-
|
| 714 |
-
|
| 715 |
-
|
| 716 |
-
pass
|
| 717 |
-
return []
|
| 718 |
|
| 719 |
def get_ai_parser(lines_or_sample):
|
| 720 |
"""
|
| 721 |
Dynamically generates or extracts a bank statement parser using AI.
|
| 722 |
-
Scans at least 2 pages prior of input lines
|
|
|
|
| 723 |
"""
|
| 724 |
if isinstance(lines_or_sample, list):
|
| 725 |
sample_text = get_multi_page_sample(lines_or_sample)
|
|
@@ -731,15 +746,16 @@ def get_ai_parser(lines_or_sample):
|
|
| 731 |
prompt = f"""You are an expert Python developer and financial statement parser.
|
| 732 |
Below are sample text lines extracted from Page 1 and Page 2 of a bank statement:
|
| 733 |
|
| 734 |
-
--- STATEMENT SAMPLE ---
|
| 735 |
{sample_text[:2500]}
|
| 736 |
--- END SAMPLE ---
|
| 737 |
|
| 738 |
-
Write a standalone Python function `parse_statement(lines)`
|
|
|
|
| 739 |
Output ONLY valid Python code inside a ```python ``` markdown code block:
|
| 740 |
```python
|
| 741 |
def parse_statement(lines):
|
| 742 |
-
# your Python code here
|
| 743 |
return transactions
|
| 744 |
```
|
| 745 |
"""
|
|
@@ -786,16 +802,16 @@ def parse_statement(lines):
|
|
| 786 |
except Exception as e:
|
| 787 |
provider_errors.append(f"{provider['name']} code gen: {e}")
|
| 788 |
|
| 789 |
-
# Phase 2: Direct LLM Extraction Fallback
|
| 790 |
for provider in PROVIDERS:
|
| 791 |
if not provider.get("api_key"):
|
| 792 |
continue
|
| 793 |
try:
|
| 794 |
-
direct_txs = extract_direct_transactions_from_llm(
|
| 795 |
if direct_txs:
|
| 796 |
target_prov = provider
|
| 797 |
def direct_ai_fn(input_lines):
|
| 798 |
-
res = extract_direct_transactions_from_llm(
|
| 799 |
return res if res else direct_txs
|
| 800 |
return direct_ai_fn, f"Other Bank ({target_prov['name'].upper()} Direct AI)"
|
| 801 |
except Exception as e:
|
|
|
|
| 671 |
|
| 672 |
return "\n".join(sample)
|
| 673 |
|
| 674 |
+
def extract_direct_transactions_from_llm(all_lines_input, provider):
|
| 675 |
"""
|
| 676 |
Direct LLM transaction extractor as a fail-safe fallback.
|
| 677 |
+
Processes ALL lines across ALL pages of the PDF document in page chunks.
|
| 678 |
"""
|
| 679 |
+
if isinstance(all_lines_input, list):
|
| 680 |
+
lines_list = all_lines_input
|
| 681 |
+
else:
|
| 682 |
+
lines_list = str(all_lines_input).split("\n")
|
| 683 |
+
|
| 684 |
+
if not lines_list:
|
| 685 |
+
return []
|
| 686 |
+
|
| 687 |
+
all_transactions = []
|
| 688 |
+
chunk_size = 70
|
| 689 |
+
for i in range(0, len(lines_list), chunk_size):
|
| 690 |
+
chunk_lines = lines_list[i:i+chunk_size]
|
| 691 |
+
chunk_str = "\n".join(chunk_lines).strip()
|
| 692 |
+
if len(re.findall(r'[a-zA-Z0-9]', chunk_str)) < 20:
|
| 693 |
+
continue
|
| 694 |
+
|
| 695 |
+
prompt = f"""You are a financial bank statement parser engine.
|
| 696 |
+
Extract all transaction rows from the statement lines below:
|
| 697 |
|
| 698 |
--- STATEMENT LINES ---
|
| 699 |
+
{chunk_str[:2500]}
|
| 700 |
--- END STATEMENT LINES ---
|
| 701 |
|
| 702 |
Return ONLY a valid JSON array of objects with keys: "date", "description", "debit", "credit", "balance".
|
|
|
|
| 707 |
]
|
| 708 |
```
|
| 709 |
"""
|
| 710 |
+
try:
|
| 711 |
+
raw_text = call_llm(provider, prompt, max_tokens=1000)
|
| 712 |
+
if not raw_text:
|
| 713 |
+
continue
|
| 714 |
+
json_match = re.search(r"```(?:json)?\s*(\[\s*[\s\S]*?\])\s*```", raw_text, re.IGNORECASE)
|
| 715 |
+
if not json_match:
|
| 716 |
+
json_match = re.search(r"\[\s*[\s\S]*?\]", raw_text)
|
| 717 |
+
if json_match:
|
| 718 |
+
raw_json = json_match.group(1 if json_match.groups() else 0).strip()
|
| 719 |
+
items = json.loads(raw_json)
|
| 720 |
+
for item in items:
|
| 721 |
+
if isinstance(item, dict) and item.get("date"):
|
| 722 |
+
all_transactions.append({
|
| 723 |
+
"date": str(item.get("date") or ""),
|
| 724 |
+
"description": str(item.get("description") or ""),
|
| 725 |
+
"debit": to_float(item.get("debit")),
|
| 726 |
+
"credit": to_float(item.get("credit")),
|
| 727 |
+
"balance": to_float(item.get("balance")),
|
| 728 |
+
})
|
| 729 |
+
except Exception:
|
| 730 |
+
pass
|
| 731 |
+
return all_transactions
|
|
|
|
|
|
|
| 732 |
|
| 733 |
def get_ai_parser(lines_or_sample):
|
| 734 |
"""
|
| 735 |
Dynamically generates or extracts a bank statement parser using AI.
|
| 736 |
+
Scans at least 2 pages prior of input lines for pattern learning,
|
| 737 |
+
and returns a parser that processes ALL pages of the entire document.
|
| 738 |
"""
|
| 739 |
if isinstance(lines_or_sample, list):
|
| 740 |
sample_text = get_multi_page_sample(lines_or_sample)
|
|
|
|
| 746 |
prompt = f"""You are an expert Python developer and financial statement parser.
|
| 747 |
Below are sample text lines extracted from Page 1 and Page 2 of a bank statement:
|
| 748 |
|
| 749 |
+
--- STATEMENT SAMPLE (PAGES 1 & 2) ---
|
| 750 |
{sample_text[:2500]}
|
| 751 |
--- END SAMPLE ---
|
| 752 |
|
| 753 |
+
Write a standalone Python function `parse_statement(lines)` that will be called on `lines` containing ALL lines of the full multi-page statement document (Page 1 through Page N).
|
| 754 |
+
Your function MUST iterate through all lines in `lines` and return a list of dicts with keys 'date', 'description', 'debit', 'credit', 'balance'.
|
| 755 |
Output ONLY valid Python code inside a ```python ``` markdown code block:
|
| 756 |
```python
|
| 757 |
def parse_statement(lines):
|
| 758 |
+
# your Python code here (must loop over all lines in lines)
|
| 759 |
return transactions
|
| 760 |
```
|
| 761 |
"""
|
|
|
|
| 802 |
except Exception as e:
|
| 803 |
provider_errors.append(f"{provider['name']} code gen: {e}")
|
| 804 |
|
| 805 |
+
# Phase 2: Direct LLM Extraction Fallback (Processes ALL pages/lines)
|
| 806 |
for provider in PROVIDERS:
|
| 807 |
if not provider.get("api_key"):
|
| 808 |
continue
|
| 809 |
try:
|
| 810 |
+
direct_txs = extract_direct_transactions_from_llm(all_lines, provider)
|
| 811 |
if direct_txs:
|
| 812 |
target_prov = provider
|
| 813 |
def direct_ai_fn(input_lines):
|
| 814 |
+
res = extract_direct_transactions_from_llm(input_lines, target_prov)
|
| 815 |
return res if res else direct_txs
|
| 816 |
return direct_ai_fn, f"Other Bank ({target_prov['name'].upper()} Direct AI)"
|
| 817 |
except Exception as e:
|