import os import re import zipfile import tempfile import multiprocessing from io import BytesIO import pandas as pd try: import pymupdf as fitz except ImportError: import fitz from PIL import Image from groq import Groq from docx import Document from docx.shared import Pt from docx.enum.text import WD_ALIGN_PARAGRAPH from dotenv import load_dotenv from pypdf import PdfReader, PdfWriter import ocrmypdf load_dotenv() GROQ_KEY = os.environ.get("GROQ_API_KEY") _client = Groq(api_key=GROQ_KEY) if GROQ_KEY and GROQ_KEY != "your-groq-api-key-here" else None def _groq(): if _client is None: raise ValueError( "GROQ_API_KEY is not set or still has the placeholder value. " "Add your real API key in the .env file." ) return _client # ── helpers ──────────────────────────────────────────────────────────────── def _cb(progress, message, callback): if callback: callback(progress, message) # ── PDF split (single PDF → ZIP of page-PDFs) ────────────────────────────── def pdf_split_process(file_bytes: bytes, split_size: int = 1, progress_callback=None): _cb(5, "Reading PDF…", progress_callback) doc = fitz.open(stream=file_bytes, filetype="pdf") total = len(doc) _cb(10, f"Splitting {total} page(s) into groups of {split_size}…", progress_callback) zip_buf = BytesIO() with zipfile.ZipFile(zip_buf, "w", zipfile.ZIP_DEFLATED) as zf: for start_idx in range(0, total, split_size): end_idx = min(start_idx + split_size - 1, total - 1) pct = 10 + int(80 * (end_idx + 1) / total) if start_idx == end_idx: _cb(pct, f"Processing page {start_idx + 1} of {total}…", progress_callback) else: _cb(pct, f"Processing pages {start_idx + 1} to {end_idx + 1} of {total}…", progress_callback) page_doc = fitz.open() page_doc.insert_pdf(doc, from_page=start_idx, to_page=end_idx) page_buf = BytesIO() page_doc.save(page_buf) page_doc.close() if start_idx == end_idx: filename = f"page_{start_idx + 1:03d}.pdf" else: filename = f"pages_{start_idx + 1:03d}-{end_idx + 1:03d}.pdf" zf.writestr(filename, page_buf.getvalue()) doc.close() _cb(100, "Split complete", progress_callback) return zip_buf.getvalue() # ── PDF merge (multiple PDFs → single PDF) ───────────────────────────────── def pdf_merge_process(files_data: list[tuple[str, bytes]], progress_callback=None): _cb(5, "Preparing merge…", progress_callback) merged = fitz.open() total_pages = 0 for fname, data in files_data: d = fitz.open(stream=data, filetype="pdf") total_pages += len(d) d.close() done = 0 for fname, data in files_data: d = fitz.open(stream=data, filetype="pdf") for i in range(len(d)): pct = 5 + int(90 * (done + 1) / total_pages) _cb(pct, f"Merging {fname} — page {i + 1}…", progress_callback) merged.insert_pdf(d, from_page=i, to_page=i) done += 1 d.close() buf = BytesIO() merged.save(buf) merged.close() _cb(100, "Merge complete", progress_callback) return buf.getvalue() # ── PDF to Word ──────────────────────────────────────────────────────────── def pdf_to_word_process(file_bytes: bytes, filename: str, progress_callback=None): """Extract PDF text into a Word (.docx) document.""" _cb(5, "Reading PDF…", progress_callback) reader = PdfReader(BytesIO(file_bytes)) total_pages = len(reader.pages) _cb(10, f"Found {total_pages} page(s). Building Word document…", progress_callback) doc = Document() doc.add_heading(f"SCANLY PDF to Word — {filename}", level=0) for idx, page in enumerate(reader.pages, start=1): pct = 10 + int(80 * idx / total_pages) _cb(pct, f"Processing page {idx} of {total_pages}…", progress_callback) doc.add_heading(f"Page {idx}", level=2) text = page.extract_text() or "" if text.strip(): doc.add_paragraph(text.strip()) else: doc.add_paragraph("[No selectable text found on this page.]") buf = BytesIO() doc.save(buf) buf.seek(0) _cb(100, "Word document ready", progress_callback) return buf.getvalue() # ── PDF Delete Pages ─────────────────────────────────────────────────────── def pdf_delete_pages_process(file_bytes: bytes, pages_to_delete: str, progress_callback=None): """Remove specified pages from a PDF and return the cleaned PDF bytes.""" _cb(5, "Reading PDF…", progress_callback) reader = PdfReader(BytesIO(file_bytes)) total_pages = len(reader.pages) _cb(10, f"PDF has {total_pages} page(s). Parsing deletion list…", progress_callback) # Parse page spec like "1,3-5,7" delete_set: set[int] = set() for part in pages_to_delete.split(","): part = part.strip() if not part: continue if "-" in part: start_str, end_str = part.split("-", 1) start = int(start_str.strip()) end = int(end_str.strip()) if start > end: start, end = end, start delete_set.update(range(start, end + 1)) else: delete_set.add(int(part)) # Validate invalid = [p for p in delete_set if p < 1 or p > total_pages] if invalid: raise ValueError(f"Page number out of range (valid: 1–{total_pages}): {invalid[0]}") if len(delete_set) >= total_pages: raise ValueError("Cannot delete every page from a PDF — at least one page must remain.") _cb(25, f"Deleting {len(delete_set)} page(s)…", progress_callback) writer = PdfWriter() for idx, page in enumerate(reader.pages, start=1): if idx not in delete_set: writer.add_page(page) buf = BytesIO() writer.write(buf) buf.seek(0) _cb(100, f"Done — {len(writer.pages)} page(s) remaining", progress_callback) return buf.getvalue() # ── Excel → CSV ──────────────────────────────────────────────────────────── def excel_to_csv_process(file_bytes: bytes, filename: str, progress_callback=None): _cb(10, "Reading spreadsheet…", progress_callback) ext = os.path.splitext(filename)[1].lower() if ext == ".csv": _cb(100, "Already CSV", progress_callback) return file_bytes if ext in (".xlsx", ".xlsm"): df = pd.read_excel(BytesIO(file_bytes), engine="openpyxl") elif ext == ".xls": df = pd.read_excel(BytesIO(file_bytes), engine="xlrd") else: raise ValueError(f"Unsupported format: {ext}") _cb(50, f"Read {len(df)} rows — writing CSV…", progress_callback) csv_buf = BytesIO() df.to_csv(csv_buf, index=False) csv_buf.seek(0) _cb(100, "Done", progress_callback) return csv_buf.getvalue() # ── Summarize (PDF / DOCX / TXT → DOCX with bullet points) ──────────────── def summarize_process(file_bytes: bytes, filename: str, progress_callback=None): _cb(5, "Reading document…", progress_callback) ext = os.path.splitext(filename)[1].lower() # ── extract plain text ── text = "" if ext == ".pdf": _cb(10, "Extracting text from PDF…", progress_callback) doc = fitz.open(stream=file_bytes, filetype="pdf") pages = [doc[i].get_text() for i in range(len(doc))] doc.close() text = "\n".join(pages) elif ext == ".docx": _cb(10, "Reading DOCX…", progress_callback) doc = Document(BytesIO(file_bytes)) text = "\n".join(p.text for p in doc.paragraphs if p.text.strip()) elif ext == ".txt": _cb(10, "Reading text file…", progress_callback) text = file_bytes.decode("utf-8", errors="replace") else: raise ValueError(f"Unsupported format: {ext}") text = text.strip() if not text: raise ValueError("No text could be extracted from the document.") _cb(30, f"Extracted {len(text)} characters — summarising with AI…", progress_callback) resp = _groq().chat.completions.create( model="llama-3.3-70b-versatile", messages=[ { "role": "system", "content": ( "You are a professional document summariser. " "Summarise the following document in clean, well-organised bullet points. " "Use short section headers with emoji where appropriate. " "Cover key topics, main arguments, and important data. " "Output in plain text using Markdown-style bullet points." ), }, {"role": "user", "content": text[:32000]}, ], temperature=0.3, max_tokens=4096, ) summary = resp.choices[0].message.content _cb(70, "Generating DOCX…", progress_callback) doc_out = Document() style = doc_out.styles["Normal"] style.font.name = "Calibri" style.font.size = Pt(11) doc_out.add_heading("Summary", level=1) for line in summary.split("\n"): line = line.strip() if line: doc_out.add_paragraph(line) buf = BytesIO() doc_out.save(buf) buf.seek(0) _cb(100, "Done", progress_callback) return buf.getvalue() # ── PDF OCR (local ocrmypdf) ─────────────────────────────────────────────── def _setup_tesseract_env(): tesseract_paths = [ r"C:\Program Files\Tesseract-OCR", r"C:\Program Files (x86)\Tesseract-OCR", os.path.expanduser(r"~\AppData\Local\Programs\Tesseract-OCR"), ] for t_path in tesseract_paths: if os.path.exists(t_path) and t_path not in os.environ.get("PATH", ""): os.environ["PATH"] = t_path + ";" + os.environ.get("PATH", "") _setup_tesseract_env() def _ocr_options(mode): workers = max(1, multiprocessing.cpu_count() or 4) if mode == "accurate": return { "force_ocr": True, "deskew": True, "rotate_pages": True, "oversample": 300, "optimize": 1, "jobs": workers, "progress_bar": False, } return { "skip_text": True, "deskew": False, "rotate_pages": False, "oversample": 150, "optimize": 0, "jobs": workers, "tesseract_timeout": 30, "progress_bar": False, } def pdf_ocr_process(file_bytes: bytes, filename: str, mode: str = "fast", progress_callback=None): _cb(10, "Writing temporary file…", progress_callback) ext = os.path.splitext(filename)[1].lower() if ext != ".pdf": raise ValueError("Searchable PDF OCR requires a PDF file.") input_file = tempfile.NamedTemporaryFile(suffix=".pdf", delete=False) input_file.write(file_bytes) input_file.close() output_file = tempfile.NamedTemporaryFile(suffix=".pdf", delete=False) output_file.close() try: _cb(30, f"Running ocrmypdf OCR ({mode} mode)…", progress_callback) ocrmypdf.ocr( input_file.name, output_file.name, **_ocr_options(mode) ) _cb(90, "Reading OCR output…", progress_callback) with open(output_file.name, "rb") as f: result_bytes = f.read() _cb(100, "Done", progress_callback) return result_bytes finally: for path in (input_file.name, output_file.name): try: os.remove(path) except OSError: pass def bank_statement_process(file_input, bank_slug: str, page_numbers: str = None, password: str = None, progress_callback=None): from statement_parser import parse_statement_pdf, excel_bytes_from_df, excel_bytes_from_multi_statements import pandas as pd # Check if single file bytes or batch list of (filename, bytes) if isinstance(file_input, (bytes, bytearray)): file_inputs = [("statement.pdf", file_input)] elif isinstance(file_input, list): file_inputs = file_input else: file_inputs = [("statement.pdf", file_input)] total_files = len(file_inputs) if total_files == 1: fname, fbytes = file_inputs[0] _cb(10, f"Reading {fname}…", progress_callback) df = parse_statement_pdf(fbytes, bank_slug=bank_slug, page_numbers=page_numbers, password=password) _cb(60, f"Parsed {len(df)} transactions. Building Excel workbook…", progress_callback) output = excel_bytes_from_df(df) _cb(100, "Done", progress_callback) return output.getvalue() all_dfs = [] for idx, (fname, fbytes) in enumerate(file_inputs): pct = int(10 + (idx / total_files) * 75) _cb(pct, f"Parsing file {idx + 1} of {total_files}: {fname}…", progress_callback) try: df = parse_statement_pdf(fbytes, bank_slug=bank_slug, page_numbers=page_numbers, password=password) if df is not None and not df.empty: df_with_file = df.copy() if "statement_file" not in df_with_file.columns: df_with_file.insert(0, "statement_file", fname) all_dfs.append((fname, df_with_file)) except Exception: continue if not all_dfs: raise ValueError("Could not extract any transactions from the uploaded statements.") total_txs = sum(len(d) for _, d in all_dfs) _cb(90, f"Consolidating {total_txs} transactions across {len(all_dfs)} statements…", progress_callback) combined_df = pd.concat([d for _, d in all_dfs], ignore_index=True) output = excel_bytes_from_multi_statements(all_dfs, combined_df) _cb(100, "Done", progress_callback) return output.getvalue()