VsinghiAssociates commited on
Commit
a0e793b
·
verified ·
1 Parent(s): 8227cf3

Upload folder using huggingface_hub

Browse files
Files changed (1) hide show
  1. statement_parser.py +52 -36
statement_parser.py CHANGED
@@ -671,16 +671,32 @@ def get_multi_page_sample(lines_input, lines_per_page=35):
671
 
672
  return "\n".join(sample)
673
 
674
- def extract_direct_transactions_from_llm(sample_text_str, provider):
675
  """
676
  Direct LLM transaction extractor as a fail-safe fallback.
677
- Extracts transactions directly into a clean JSON array [ {...}, {...} ].
678
  """
679
- prompt = f"""You are a financial bank statement parser engine.
680
- Extract all transaction rows from the statement lines below (spanning Pages 1 & 2):
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
681
 
682
  --- STATEMENT LINES ---
683
- {sample_text_str[:2500]}
684
  --- END STATEMENT LINES ---
685
 
686
  Return ONLY a valid JSON array of objects with keys: "date", "description", "debit", "credit", "balance".
@@ -691,35 +707,34 @@ Output ONLY inside a ```json ``` code block:
691
  ]
692
  ```
693
  """
694
- try:
695
- raw_text = call_llm(provider, prompt, max_tokens=1000)
696
- if not raw_text:
697
- return []
698
- json_match = re.search(r"```(?:json)?\s*(\[\s*[\s\S]*?\])\s*```", raw_text, re.IGNORECASE)
699
- if not json_match:
700
- json_match = re.search(r"\[\s*[\s\S]*?\]", raw_text)
701
- if json_match:
702
- raw_json = json_match.group(1 if json_match.groups() else 0).strip()
703
- items = json.loads(raw_json)
704
- cleaned = []
705
- for item in items:
706
- if isinstance(item, dict) and item.get("date"):
707
- cleaned.append({
708
- "date": str(item.get("date") or ""),
709
- "description": str(item.get("description") or ""),
710
- "debit": to_float(item.get("debit")),
711
- "credit": to_float(item.get("credit")),
712
- "balance": to_float(item.get("balance")),
713
- })
714
- return cleaned
715
- except Exception:
716
- pass
717
- return []
718
 
719
  def get_ai_parser(lines_or_sample):
720
  """
721
  Dynamically generates or extracts a bank statement parser using AI.
722
- Scans at least 2 pages prior of input lines with optimized low token consumption.
 
723
  """
724
  if isinstance(lines_or_sample, list):
725
  sample_text = get_multi_page_sample(lines_or_sample)
@@ -731,15 +746,16 @@ def get_ai_parser(lines_or_sample):
731
  prompt = f"""You are an expert Python developer and financial statement parser.
732
  Below are sample text lines extracted from Page 1 and Page 2 of a bank statement:
733
 
734
- --- STATEMENT SAMPLE ---
735
  {sample_text[:2500]}
736
  --- END SAMPLE ---
737
 
738
- Write a standalone Python function `parse_statement(lines)` returning a list of dicts with keys 'date', 'description', 'debit', 'credit', 'balance'.
 
739
  Output ONLY valid Python code inside a ```python ``` markdown code block:
740
  ```python
741
  def parse_statement(lines):
742
- # your Python code here
743
  return transactions
744
  ```
745
  """
@@ -786,16 +802,16 @@ def parse_statement(lines):
786
  except Exception as e:
787
  provider_errors.append(f"{provider['name']} code gen: {e}")
788
 
789
- # Phase 2: Direct LLM Extraction Fallback
790
  for provider in PROVIDERS:
791
  if not provider.get("api_key"):
792
  continue
793
  try:
794
- direct_txs = extract_direct_transactions_from_llm(sample_text, provider)
795
  if direct_txs:
796
  target_prov = provider
797
  def direct_ai_fn(input_lines):
798
- res = extract_direct_transactions_from_llm("\n".join(input_lines), target_prov)
799
  return res if res else direct_txs
800
  return direct_ai_fn, f"Other Bank ({target_prov['name'].upper()} Direct AI)"
801
  except Exception as e:
 
671
 
672
  return "\n".join(sample)
673
 
674
+ def extract_direct_transactions_from_llm(all_lines_input, provider):
675
  """
676
  Direct LLM transaction extractor as a fail-safe fallback.
677
+ Processes ALL lines across ALL pages of the PDF document in page chunks.
678
  """
679
+ if isinstance(all_lines_input, list):
680
+ lines_list = all_lines_input
681
+ else:
682
+ lines_list = str(all_lines_input).split("\n")
683
+
684
+ if not lines_list:
685
+ return []
686
+
687
+ all_transactions = []
688
+ chunk_size = 70
689
+ for i in range(0, len(lines_list), chunk_size):
690
+ chunk_lines = lines_list[i:i+chunk_size]
691
+ chunk_str = "\n".join(chunk_lines).strip()
692
+ if len(re.findall(r'[a-zA-Z0-9]', chunk_str)) < 20:
693
+ continue
694
+
695
+ prompt = f"""You are a financial bank statement parser engine.
696
+ Extract all transaction rows from the statement lines below:
697
 
698
  --- STATEMENT LINES ---
699
+ {chunk_str[:2500]}
700
  --- END STATEMENT LINES ---
701
 
702
  Return ONLY a valid JSON array of objects with keys: "date", "description", "debit", "credit", "balance".
 
707
  ]
708
  ```
709
  """
710
+ try:
711
+ raw_text = call_llm(provider, prompt, max_tokens=1000)
712
+ if not raw_text:
713
+ continue
714
+ json_match = re.search(r"```(?:json)?\s*(\[\s*[\s\S]*?\])\s*```", raw_text, re.IGNORECASE)
715
+ if not json_match:
716
+ json_match = re.search(r"\[\s*[\s\S]*?\]", raw_text)
717
+ if json_match:
718
+ raw_json = json_match.group(1 if json_match.groups() else 0).strip()
719
+ items = json.loads(raw_json)
720
+ for item in items:
721
+ if isinstance(item, dict) and item.get("date"):
722
+ all_transactions.append({
723
+ "date": str(item.get("date") or ""),
724
+ "description": str(item.get("description") or ""),
725
+ "debit": to_float(item.get("debit")),
726
+ "credit": to_float(item.get("credit")),
727
+ "balance": to_float(item.get("balance")),
728
+ })
729
+ except Exception:
730
+ pass
731
+ return all_transactions
 
 
732
 
733
  def get_ai_parser(lines_or_sample):
734
  """
735
  Dynamically generates or extracts a bank statement parser using AI.
736
+ Scans at least 2 pages prior of input lines for pattern learning,
737
+ and returns a parser that processes ALL pages of the entire document.
738
  """
739
  if isinstance(lines_or_sample, list):
740
  sample_text = get_multi_page_sample(lines_or_sample)
 
746
  prompt = f"""You are an expert Python developer and financial statement parser.
747
  Below are sample text lines extracted from Page 1 and Page 2 of a bank statement:
748
 
749
+ --- STATEMENT SAMPLE (PAGES 1 & 2) ---
750
  {sample_text[:2500]}
751
  --- END SAMPLE ---
752
 
753
+ Write a standalone Python function `parse_statement(lines)` that will be called on `lines` containing ALL lines of the full multi-page statement document (Page 1 through Page N).
754
+ Your function MUST iterate through all lines in `lines` and return a list of dicts with keys 'date', 'description', 'debit', 'credit', 'balance'.
755
  Output ONLY valid Python code inside a ```python ``` markdown code block:
756
  ```python
757
  def parse_statement(lines):
758
+ # your Python code here (must loop over all lines in lines)
759
  return transactions
760
  ```
761
  """
 
802
  except Exception as e:
803
  provider_errors.append(f"{provider['name']} code gen: {e}")
804
 
805
+ # Phase 2: Direct LLM Extraction Fallback (Processes ALL pages/lines)
806
  for provider in PROVIDERS:
807
  if not provider.get("api_key"):
808
  continue
809
  try:
810
+ direct_txs = extract_direct_transactions_from_llm(all_lines, provider)
811
  if direct_txs:
812
  target_prov = provider
813
  def direct_ai_fn(input_lines):
814
+ res = extract_direct_transactions_from_llm(input_lines, target_prov)
815
  return res if res else direct_txs
816
  return direct_ai_fn, f"Other Bank ({target_prov['name'].upper()} Direct AI)"
817
  except Exception as e: