Spaces:
Sleeping
Sleeping
Download services.py from VsinghiAssociates/Scanlly: direct link, hf CLI and curl.
- Browser
- Download file 15.1 kB
-
https://huggingface.co/spaces/VsinghiAssociates/Scanlly/resolve/main/services.py
- Command line
-
hf download hf://spaces/VsinghiAssociates/Scanlly/services.py
-
curl -L -o services.py https://huggingface.co/spaces/VsinghiAssociates/Scanlly/resolve/main/services.py
15.1 kB
| import os | |
| import re | |
| import zipfile | |
| import tempfile | |
| import multiprocessing | |
| from io import BytesIO | |
| import pandas as pd | |
| try: | |
| import pymupdf as fitz | |
| except ImportError: | |
| import fitz | |
| from PIL import Image | |
| from groq import Groq | |
| from docx import Document | |
| from docx.shared import Pt | |
| from docx.enum.text import WD_ALIGN_PARAGRAPH | |
| from dotenv import load_dotenv | |
| from pypdf import PdfReader, PdfWriter | |
| import ocrmypdf | |
| load_dotenv() | |
| GROQ_KEY = os.environ.get("GROQ_API_KEY") | |
| _client = Groq(api_key=GROQ_KEY) if GROQ_KEY and GROQ_KEY != "your-groq-api-key-here" else None | |
| def _groq(): | |
| if _client is None: | |
| raise ValueError( | |
| "GROQ_API_KEY is not set or still has the placeholder value. " | |
| "Add your real API key in the .env file." | |
| ) | |
| return _client | |
| # ββ helpers ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| def _cb(progress, message, callback): | |
| if callback: | |
| callback(progress, message) | |
| # ββ PDF split (single PDF β ZIP of page-PDFs) ββββββββββββββββββββββββββββββ | |
| def pdf_split_process(file_bytes: bytes, split_size: int = 1, progress_callback=None): | |
| _cb(5, "Reading PDFβ¦", progress_callback) | |
| doc = fitz.open(stream=file_bytes, filetype="pdf") | |
| total = len(doc) | |
| _cb(10, f"Splitting {total} page(s) into groups of {split_size}β¦", progress_callback) | |
| zip_buf = BytesIO() | |
| with zipfile.ZipFile(zip_buf, "w", zipfile.ZIP_DEFLATED) as zf: | |
| for start_idx in range(0, total, split_size): | |
| end_idx = min(start_idx + split_size - 1, total - 1) | |
| pct = 10 + int(80 * (end_idx + 1) / total) | |
| if start_idx == end_idx: | |
| _cb(pct, f"Processing page {start_idx + 1} of {total}β¦", progress_callback) | |
| else: | |
| _cb(pct, f"Processing pages {start_idx + 1} to {end_idx + 1} of {total}β¦", progress_callback) | |
| page_doc = fitz.open() | |
| page_doc.insert_pdf(doc, from_page=start_idx, to_page=end_idx) | |
| page_buf = BytesIO() | |
| page_doc.save(page_buf) | |
| page_doc.close() | |
| if start_idx == end_idx: | |
| filename = f"page_{start_idx + 1:03d}.pdf" | |
| else: | |
| filename = f"pages_{start_idx + 1:03d}-{end_idx + 1:03d}.pdf" | |
| zf.writestr(filename, page_buf.getvalue()) | |
| doc.close() | |
| _cb(100, "Split complete", progress_callback) | |
| return zip_buf.getvalue() | |
| # ββ PDF merge (multiple PDFs β single PDF) βββββββββββββββββββββββββββββββββ | |
| def pdf_merge_process(files_data: list[tuple[str, bytes]], progress_callback=None): | |
| _cb(5, "Preparing mergeβ¦", progress_callback) | |
| merged = fitz.open() | |
| total_pages = 0 | |
| for fname, data in files_data: | |
| d = fitz.open(stream=data, filetype="pdf") | |
| total_pages += len(d) | |
| d.close() | |
| done = 0 | |
| for fname, data in files_data: | |
| d = fitz.open(stream=data, filetype="pdf") | |
| for i in range(len(d)): | |
| pct = 5 + int(90 * (done + 1) / total_pages) | |
| _cb(pct, f"Merging {fname} β page {i + 1}β¦", progress_callback) | |
| merged.insert_pdf(d, from_page=i, to_page=i) | |
| done += 1 | |
| d.close() | |
| buf = BytesIO() | |
| merged.save(buf) | |
| merged.close() | |
| _cb(100, "Merge complete", progress_callback) | |
| return buf.getvalue() | |
| # ββ PDF to Word ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| def pdf_to_word_process(file_bytes: bytes, filename: str, progress_callback=None): | |
| """Extract PDF text into a Word (.docx) document.""" | |
| _cb(5, "Reading PDFβ¦", progress_callback) | |
| reader = PdfReader(BytesIO(file_bytes)) | |
| total_pages = len(reader.pages) | |
| _cb(10, f"Found {total_pages} page(s). Building Word documentβ¦", progress_callback) | |
| doc = Document() | |
| doc.add_heading(f"SCANLY PDF to Word β {filename}", level=0) | |
| for idx, page in enumerate(reader.pages, start=1): | |
| pct = 10 + int(80 * idx / total_pages) | |
| _cb(pct, f"Processing page {idx} of {total_pages}β¦", progress_callback) | |
| doc.add_heading(f"Page {idx}", level=2) | |
| text = page.extract_text() or "" | |
| if text.strip(): | |
| doc.add_paragraph(text.strip()) | |
| else: | |
| doc.add_paragraph("[No selectable text found on this page.]") | |
| buf = BytesIO() | |
| doc.save(buf) | |
| buf.seek(0) | |
| _cb(100, "Word document ready", progress_callback) | |
| return buf.getvalue() | |
| # ββ PDF Delete Pages βββββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| def pdf_delete_pages_process(file_bytes: bytes, pages_to_delete: str, progress_callback=None): | |
| """Remove specified pages from a PDF and return the cleaned PDF bytes.""" | |
| _cb(5, "Reading PDFβ¦", progress_callback) | |
| reader = PdfReader(BytesIO(file_bytes)) | |
| total_pages = len(reader.pages) | |
| _cb(10, f"PDF has {total_pages} page(s). Parsing deletion listβ¦", progress_callback) | |
| # Parse page spec like "1,3-5,7" | |
| delete_set: set[int] = set() | |
| for part in pages_to_delete.split(","): | |
| part = part.strip() | |
| if not part: | |
| continue | |
| if "-" in part: | |
| start_str, end_str = part.split("-", 1) | |
| start = int(start_str.strip()) | |
| end = int(end_str.strip()) | |
| if start > end: | |
| start, end = end, start | |
| delete_set.update(range(start, end + 1)) | |
| else: | |
| delete_set.add(int(part)) | |
| # Validate | |
| invalid = [p for p in delete_set if p < 1 or p > total_pages] | |
| if invalid: | |
| raise ValueError(f"Page number out of range (valid: 1β{total_pages}): {invalid[0]}") | |
| if len(delete_set) >= total_pages: | |
| raise ValueError("Cannot delete every page from a PDF β at least one page must remain.") | |
| _cb(25, f"Deleting {len(delete_set)} page(s)β¦", progress_callback) | |
| writer = PdfWriter() | |
| for idx, page in enumerate(reader.pages, start=1): | |
| if idx not in delete_set: | |
| writer.add_page(page) | |
| buf = BytesIO() | |
| writer.write(buf) | |
| buf.seek(0) | |
| _cb(100, f"Done β {len(writer.pages)} page(s) remaining", progress_callback) | |
| return buf.getvalue() | |
| # ββ Excel β CSV ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| def excel_to_csv_process(file_bytes: bytes, filename: str, progress_callback=None): | |
| _cb(10, "Reading spreadsheetβ¦", progress_callback) | |
| ext = os.path.splitext(filename)[1].lower() | |
| if ext == ".csv": | |
| _cb(100, "Already CSV", progress_callback) | |
| return file_bytes | |
| if ext in (".xlsx", ".xlsm"): | |
| df = pd.read_excel(BytesIO(file_bytes), engine="openpyxl") | |
| elif ext == ".xls": | |
| df = pd.read_excel(BytesIO(file_bytes), engine="xlrd") | |
| else: | |
| raise ValueError(f"Unsupported format: {ext}") | |
| _cb(50, f"Read {len(df)} rows β writing CSVβ¦", progress_callback) | |
| csv_buf = BytesIO() | |
| df.to_csv(csv_buf, index=False) | |
| csv_buf.seek(0) | |
| _cb(100, "Done", progress_callback) | |
| return csv_buf.getvalue() | |
| # ββ Summarize (PDF / DOCX / TXT β DOCX with bullet points) ββββββββββββββββ | |
| def summarize_process(file_bytes: bytes, filename: str, progress_callback=None): | |
| _cb(5, "Reading documentβ¦", progress_callback) | |
| ext = os.path.splitext(filename)[1].lower() | |
| # ββ extract plain text ββ | |
| text = "" | |
| if ext == ".pdf": | |
| _cb(10, "Extracting text from PDFβ¦", progress_callback) | |
| doc = fitz.open(stream=file_bytes, filetype="pdf") | |
| pages = [doc[i].get_text() for i in range(len(doc))] | |
| doc.close() | |
| text = "\n".join(pages) | |
| elif ext == ".docx": | |
| _cb(10, "Reading DOCXβ¦", progress_callback) | |
| doc = Document(BytesIO(file_bytes)) | |
| text = "\n".join(p.text for p in doc.paragraphs if p.text.strip()) | |
| elif ext == ".txt": | |
| _cb(10, "Reading text fileβ¦", progress_callback) | |
| text = file_bytes.decode("utf-8", errors="replace") | |
| else: | |
| raise ValueError(f"Unsupported format: {ext}") | |
| text = text.strip() | |
| if not text: | |
| raise ValueError("No text could be extracted from the document.") | |
| _cb(30, f"Extracted {len(text)} characters β summarising with AIβ¦", progress_callback) | |
| resp = _groq().chat.completions.create( | |
| model="llama-3.3-70b-versatile", | |
| messages=[ | |
| { | |
| "role": "system", | |
| "content": ( | |
| "You are a professional document summariser. " | |
| "Summarise the following document in clean, well-organised bullet points. " | |
| "Use short section headers with emoji where appropriate. " | |
| "Cover key topics, main arguments, and important data. " | |
| "Output in plain text using Markdown-style bullet points." | |
| ), | |
| }, | |
| {"role": "user", "content": text[:32000]}, | |
| ], | |
| temperature=0.3, | |
| max_tokens=4096, | |
| ) | |
| summary = resp.choices[0].message.content | |
| _cb(70, "Generating DOCXβ¦", progress_callback) | |
| doc_out = Document() | |
| style = doc_out.styles["Normal"] | |
| style.font.name = "Calibri" | |
| style.font.size = Pt(11) | |
| doc_out.add_heading("Summary", level=1) | |
| for line in summary.split("\n"): | |
| line = line.strip() | |
| if line: | |
| doc_out.add_paragraph(line) | |
| buf = BytesIO() | |
| doc_out.save(buf) | |
| buf.seek(0) | |
| _cb(100, "Done", progress_callback) | |
| return buf.getvalue() | |
| # ββ PDF OCR (local ocrmypdf) βββββββββββββββββββββββββββββββββββββββββββββββ | |
| def _setup_tesseract_env(): | |
| tesseract_paths = [ | |
| r"C:\Program Files\Tesseract-OCR", | |
| r"C:\Program Files (x86)\Tesseract-OCR", | |
| os.path.expanduser(r"~\AppData\Local\Programs\Tesseract-OCR"), | |
| ] | |
| for t_path in tesseract_paths: | |
| if os.path.exists(t_path) and t_path not in os.environ.get("PATH", ""): | |
| os.environ["PATH"] = t_path + ";" + os.environ.get("PATH", "") | |
| _setup_tesseract_env() | |
| def _ocr_options(mode): | |
| workers = max(1, multiprocessing.cpu_count() or 4) | |
| if mode == "accurate": | |
| return { | |
| "force_ocr": True, | |
| "deskew": True, | |
| "rotate_pages": True, | |
| "oversample": 300, | |
| "optimize": 1, | |
| "jobs": workers, | |
| "progress_bar": False, | |
| } | |
| return { | |
| "skip_text": True, | |
| "deskew": False, | |
| "rotate_pages": False, | |
| "oversample": 150, | |
| "optimize": 0, | |
| "jobs": workers, | |
| "tesseract_timeout": 30, | |
| "progress_bar": False, | |
| } | |
| def pdf_ocr_process(file_bytes: bytes, filename: str, mode: str = "fast", progress_callback=None): | |
| _cb(10, "Writing temporary fileβ¦", progress_callback) | |
| ext = os.path.splitext(filename)[1].lower() | |
| if ext != ".pdf": | |
| raise ValueError("Searchable PDF OCR requires a PDF file.") | |
| input_file = tempfile.NamedTemporaryFile(suffix=".pdf", delete=False) | |
| input_file.write(file_bytes) | |
| input_file.close() | |
| output_file = tempfile.NamedTemporaryFile(suffix=".pdf", delete=False) | |
| output_file.close() | |
| try: | |
| _cb(30, f"Running ocrmypdf OCR ({mode} mode)β¦", progress_callback) | |
| ocrmypdf.ocr( | |
| input_file.name, | |
| output_file.name, | |
| **_ocr_options(mode) | |
| ) | |
| _cb(90, "Reading OCR outputβ¦", progress_callback) | |
| with open(output_file.name, "rb") as f: | |
| result_bytes = f.read() | |
| _cb(100, "Done", progress_callback) | |
| return result_bytes | |
| finally: | |
| for path in (input_file.name, output_file.name): | |
| try: | |
| os.remove(path) | |
| except OSError: | |
| pass | |
| def bank_statement_process(file_input, bank_slug: str, page_numbers: str = None, password: str = None, progress_callback=None): | |
| from statement_parser import parse_statement_pdf, excel_bytes_from_df, excel_bytes_from_multi_statements | |
| import pandas as pd | |
| # Check if single file bytes or batch list of (filename, bytes) | |
| if isinstance(file_input, (bytes, bytearray)): | |
| file_inputs = [("statement.pdf", file_input)] | |
| elif isinstance(file_input, list): | |
| file_inputs = file_input | |
| else: | |
| file_inputs = [("statement.pdf", file_input)] | |
| total_files = len(file_inputs) | |
| if total_files == 1: | |
| fname, fbytes = file_inputs[0] | |
| _cb(10, f"Reading {fname}β¦", progress_callback) | |
| df = parse_statement_pdf(fbytes, bank_slug=bank_slug, page_numbers=page_numbers, password=password) | |
| _cb(60, f"Parsed {len(df)} transactions. Building Excel workbookβ¦", progress_callback) | |
| output = excel_bytes_from_df(df) | |
| _cb(100, "Done", progress_callback) | |
| return output.getvalue() | |
| all_dfs = [] | |
| for idx, (fname, fbytes) in enumerate(file_inputs): | |
| pct = int(10 + (idx / total_files) * 75) | |
| _cb(pct, f"Parsing file {idx + 1} of {total_files}: {fname}β¦", progress_callback) | |
| try: | |
| df = parse_statement_pdf(fbytes, bank_slug=bank_slug, page_numbers=page_numbers, password=password) | |
| if df is not None and not df.empty: | |
| df_with_file = df.copy() | |
| if "statement_file" not in df_with_file.columns: | |
| df_with_file.insert(0, "statement_file", fname) | |
| all_dfs.append((fname, df_with_file)) | |
| except Exception: | |
| continue | |
| if not all_dfs: | |
| raise ValueError("Could not extract any transactions from the uploaded statements.") | |
| total_txs = sum(len(d) for _, d in all_dfs) | |
| _cb(90, f"Consolidating {total_txs} transactions across {len(all_dfs)} statementsβ¦", progress_callback) | |
| combined_df = pd.concat([d for _, d in all_dfs], ignore_index=True) | |
| output = excel_bytes_from_multi_statements(all_dfs, combined_df) | |
| _cb(100, "Done", progress_callback) | |
| return output.getvalue() | |