Scanlly / services.py
VsinghiAssociates's picture
Upload folder using huggingface_hub
2d875c8 verified
Raw History Blame Contribute Delete
15.1 kB
import os
import re
import zipfile
import tempfile
import multiprocessing
from io import BytesIO
import pandas as pd
try:
import pymupdf as fitz
except ImportError:
import fitz
from PIL import Image
from groq import Groq
from docx import Document
from docx.shared import Pt
from docx.enum.text import WD_ALIGN_PARAGRAPH
from dotenv import load_dotenv
from pypdf import PdfReader, PdfWriter
import ocrmypdf
load_dotenv()
GROQ_KEY = os.environ.get("GROQ_API_KEY")
_client = Groq(api_key=GROQ_KEY) if GROQ_KEY and GROQ_KEY != "your-groq-api-key-here" else None
def _groq():
if _client is None:
raise ValueError(
"GROQ_API_KEY is not set or still has the placeholder value. "
"Add your real API key in the .env file."
)
return _client
# ── helpers ────────────────────────────────────────────────────────────────
def _cb(progress, message, callback):
if callback:
callback(progress, message)
# ── PDF split (single PDF β†’ ZIP of page-PDFs) ──────────────────────────────
def pdf_split_process(file_bytes: bytes, split_size: int = 1, progress_callback=None):
_cb(5, "Reading PDF…", progress_callback)
doc = fitz.open(stream=file_bytes, filetype="pdf")
total = len(doc)
_cb(10, f"Splitting {total} page(s) into groups of {split_size}…", progress_callback)
zip_buf = BytesIO()
with zipfile.ZipFile(zip_buf, "w", zipfile.ZIP_DEFLATED) as zf:
for start_idx in range(0, total, split_size):
end_idx = min(start_idx + split_size - 1, total - 1)
pct = 10 + int(80 * (end_idx + 1) / total)
if start_idx == end_idx:
_cb(pct, f"Processing page {start_idx + 1} of {total}…", progress_callback)
else:
_cb(pct, f"Processing pages {start_idx + 1} to {end_idx + 1} of {total}…", progress_callback)
page_doc = fitz.open()
page_doc.insert_pdf(doc, from_page=start_idx, to_page=end_idx)
page_buf = BytesIO()
page_doc.save(page_buf)
page_doc.close()
if start_idx == end_idx:
filename = f"page_{start_idx + 1:03d}.pdf"
else:
filename = f"pages_{start_idx + 1:03d}-{end_idx + 1:03d}.pdf"
zf.writestr(filename, page_buf.getvalue())
doc.close()
_cb(100, "Split complete", progress_callback)
return zip_buf.getvalue()
# ── PDF merge (multiple PDFs β†’ single PDF) ─────────────────────────────────
def pdf_merge_process(files_data: list[tuple[str, bytes]], progress_callback=None):
_cb(5, "Preparing merge…", progress_callback)
merged = fitz.open()
total_pages = 0
for fname, data in files_data:
d = fitz.open(stream=data, filetype="pdf")
total_pages += len(d)
d.close()
done = 0
for fname, data in files_data:
d = fitz.open(stream=data, filetype="pdf")
for i in range(len(d)):
pct = 5 + int(90 * (done + 1) / total_pages)
_cb(pct, f"Merging {fname} β€” page {i + 1}…", progress_callback)
merged.insert_pdf(d, from_page=i, to_page=i)
done += 1
d.close()
buf = BytesIO()
merged.save(buf)
merged.close()
_cb(100, "Merge complete", progress_callback)
return buf.getvalue()
# ── PDF to Word ────────────────────────────────────────────────────────────
def pdf_to_word_process(file_bytes: bytes, filename: str, progress_callback=None):
"""Extract PDF text into a Word (.docx) document."""
_cb(5, "Reading PDF…", progress_callback)
reader = PdfReader(BytesIO(file_bytes))
total_pages = len(reader.pages)
_cb(10, f"Found {total_pages} page(s). Building Word document…", progress_callback)
doc = Document()
doc.add_heading(f"SCANLY PDF to Word β€” {filename}", level=0)
for idx, page in enumerate(reader.pages, start=1):
pct = 10 + int(80 * idx / total_pages)
_cb(pct, f"Processing page {idx} of {total_pages}…", progress_callback)
doc.add_heading(f"Page {idx}", level=2)
text = page.extract_text() or ""
if text.strip():
doc.add_paragraph(text.strip())
else:
doc.add_paragraph("[No selectable text found on this page.]")
buf = BytesIO()
doc.save(buf)
buf.seek(0)
_cb(100, "Word document ready", progress_callback)
return buf.getvalue()
# ── PDF Delete Pages ───────────────────────────────────────────────────────
def pdf_delete_pages_process(file_bytes: bytes, pages_to_delete: str, progress_callback=None):
"""Remove specified pages from a PDF and return the cleaned PDF bytes."""
_cb(5, "Reading PDF…", progress_callback)
reader = PdfReader(BytesIO(file_bytes))
total_pages = len(reader.pages)
_cb(10, f"PDF has {total_pages} page(s). Parsing deletion list…", progress_callback)
# Parse page spec like "1,3-5,7"
delete_set: set[int] = set()
for part in pages_to_delete.split(","):
part = part.strip()
if not part:
continue
if "-" in part:
start_str, end_str = part.split("-", 1)
start = int(start_str.strip())
end = int(end_str.strip())
if start > end:
start, end = end, start
delete_set.update(range(start, end + 1))
else:
delete_set.add(int(part))
# Validate
invalid = [p for p in delete_set if p < 1 or p > total_pages]
if invalid:
raise ValueError(f"Page number out of range (valid: 1–{total_pages}): {invalid[0]}")
if len(delete_set) >= total_pages:
raise ValueError("Cannot delete every page from a PDF β€” at least one page must remain.")
_cb(25, f"Deleting {len(delete_set)} page(s)…", progress_callback)
writer = PdfWriter()
for idx, page in enumerate(reader.pages, start=1):
if idx not in delete_set:
writer.add_page(page)
buf = BytesIO()
writer.write(buf)
buf.seek(0)
_cb(100, f"Done β€” {len(writer.pages)} page(s) remaining", progress_callback)
return buf.getvalue()
# ── Excel β†’ CSV ────────────────────────────────────────────────────────────
def excel_to_csv_process(file_bytes: bytes, filename: str, progress_callback=None):
_cb(10, "Reading spreadsheet…", progress_callback)
ext = os.path.splitext(filename)[1].lower()
if ext == ".csv":
_cb(100, "Already CSV", progress_callback)
return file_bytes
if ext in (".xlsx", ".xlsm"):
df = pd.read_excel(BytesIO(file_bytes), engine="openpyxl")
elif ext == ".xls":
df = pd.read_excel(BytesIO(file_bytes), engine="xlrd")
else:
raise ValueError(f"Unsupported format: {ext}")
_cb(50, f"Read {len(df)} rows β€” writing CSV…", progress_callback)
csv_buf = BytesIO()
df.to_csv(csv_buf, index=False)
csv_buf.seek(0)
_cb(100, "Done", progress_callback)
return csv_buf.getvalue()
# ── Summarize (PDF / DOCX / TXT β†’ DOCX with bullet points) ────────────────
def summarize_process(file_bytes: bytes, filename: str, progress_callback=None):
_cb(5, "Reading document…", progress_callback)
ext = os.path.splitext(filename)[1].lower()
# ── extract plain text ──
text = ""
if ext == ".pdf":
_cb(10, "Extracting text from PDF…", progress_callback)
doc = fitz.open(stream=file_bytes, filetype="pdf")
pages = [doc[i].get_text() for i in range(len(doc))]
doc.close()
text = "\n".join(pages)
elif ext == ".docx":
_cb(10, "Reading DOCX…", progress_callback)
doc = Document(BytesIO(file_bytes))
text = "\n".join(p.text for p in doc.paragraphs if p.text.strip())
elif ext == ".txt":
_cb(10, "Reading text file…", progress_callback)
text = file_bytes.decode("utf-8", errors="replace")
else:
raise ValueError(f"Unsupported format: {ext}")
text = text.strip()
if not text:
raise ValueError("No text could be extracted from the document.")
_cb(30, f"Extracted {len(text)} characters β€” summarising with AI…", progress_callback)
resp = _groq().chat.completions.create(
model="llama-3.3-70b-versatile",
messages=[
{
"role": "system",
"content": (
"You are a professional document summariser. "
"Summarise the following document in clean, well-organised bullet points. "
"Use short section headers with emoji where appropriate. "
"Cover key topics, main arguments, and important data. "
"Output in plain text using Markdown-style bullet points."
),
},
{"role": "user", "content": text[:32000]},
],
temperature=0.3,
max_tokens=4096,
)
summary = resp.choices[0].message.content
_cb(70, "Generating DOCX…", progress_callback)
doc_out = Document()
style = doc_out.styles["Normal"]
style.font.name = "Calibri"
style.font.size = Pt(11)
doc_out.add_heading("Summary", level=1)
for line in summary.split("\n"):
line = line.strip()
if line:
doc_out.add_paragraph(line)
buf = BytesIO()
doc_out.save(buf)
buf.seek(0)
_cb(100, "Done", progress_callback)
return buf.getvalue()
# ── PDF OCR (local ocrmypdf) ───────────────────────────────────────────────
def _setup_tesseract_env():
tesseract_paths = [
r"C:\Program Files\Tesseract-OCR",
r"C:\Program Files (x86)\Tesseract-OCR",
os.path.expanduser(r"~\AppData\Local\Programs\Tesseract-OCR"),
]
for t_path in tesseract_paths:
if os.path.exists(t_path) and t_path not in os.environ.get("PATH", ""):
os.environ["PATH"] = t_path + ";" + os.environ.get("PATH", "")
_setup_tesseract_env()
def _ocr_options(mode):
workers = max(1, multiprocessing.cpu_count() or 4)
if mode == "accurate":
return {
"force_ocr": True,
"deskew": True,
"rotate_pages": True,
"oversample": 300,
"optimize": 1,
"jobs": workers,
"progress_bar": False,
}
return {
"skip_text": True,
"deskew": False,
"rotate_pages": False,
"oversample": 150,
"optimize": 0,
"jobs": workers,
"tesseract_timeout": 30,
"progress_bar": False,
}
def pdf_ocr_process(file_bytes: bytes, filename: str, mode: str = "fast", progress_callback=None):
_cb(10, "Writing temporary file…", progress_callback)
ext = os.path.splitext(filename)[1].lower()
if ext != ".pdf":
raise ValueError("Searchable PDF OCR requires a PDF file.")
input_file = tempfile.NamedTemporaryFile(suffix=".pdf", delete=False)
input_file.write(file_bytes)
input_file.close()
output_file = tempfile.NamedTemporaryFile(suffix=".pdf", delete=False)
output_file.close()
try:
_cb(30, f"Running ocrmypdf OCR ({mode} mode)…", progress_callback)
ocrmypdf.ocr(
input_file.name,
output_file.name,
**_ocr_options(mode)
)
_cb(90, "Reading OCR output…", progress_callback)
with open(output_file.name, "rb") as f:
result_bytes = f.read()
_cb(100, "Done", progress_callback)
return result_bytes
finally:
for path in (input_file.name, output_file.name):
try:
os.remove(path)
except OSError:
pass
def bank_statement_process(file_input, bank_slug: str, page_numbers: str = None, password: str = None, progress_callback=None):
from statement_parser import parse_statement_pdf, excel_bytes_from_df, excel_bytes_from_multi_statements
import pandas as pd
# Check if single file bytes or batch list of (filename, bytes)
if isinstance(file_input, (bytes, bytearray)):
file_inputs = [("statement.pdf", file_input)]
elif isinstance(file_input, list):
file_inputs = file_input
else:
file_inputs = [("statement.pdf", file_input)]
total_files = len(file_inputs)
if total_files == 1:
fname, fbytes = file_inputs[0]
_cb(10, f"Reading {fname}…", progress_callback)
df = parse_statement_pdf(fbytes, bank_slug=bank_slug, page_numbers=page_numbers, password=password)
_cb(60, f"Parsed {len(df)} transactions. Building Excel workbook…", progress_callback)
output = excel_bytes_from_df(df)
_cb(100, "Done", progress_callback)
return output.getvalue()
all_dfs = []
for idx, (fname, fbytes) in enumerate(file_inputs):
pct = int(10 + (idx / total_files) * 75)
_cb(pct, f"Parsing file {idx + 1} of {total_files}: {fname}…", progress_callback)
try:
df = parse_statement_pdf(fbytes, bank_slug=bank_slug, page_numbers=page_numbers, password=password)
if df is not None and not df.empty:
df_with_file = df.copy()
if "statement_file" not in df_with_file.columns:
df_with_file.insert(0, "statement_file", fname)
all_dfs.append((fname, df_with_file))
except Exception:
continue
if not all_dfs:
raise ValueError("Could not extract any transactions from the uploaded statements.")
total_txs = sum(len(d) for _, d in all_dfs)
_cb(90, f"Consolidating {total_txs} transactions across {len(all_dfs)} statements…", progress_callback)
combined_df = pd.concat([d for _, d in all_dfs], ignore_index=True)
output = excel_bytes_from_multi_statements(all_dfs, combined_df)
_cb(100, "Done", progress_callback)
return output.getvalue()