Spaces:
Sleeping
Sleeping
| """ | |
| services/postprocess.py — AI post-processing utilities for OCR output. | |
| Functions: | |
| - clean_text() : Remove noise/artifacts from raw OCR text | |
| - summarize() : Return first 1–2 sentences as summary | |
| - detect_headings(): Detect uppercase or heading-like lines | |
| """ | |
| import re | |
| from typing import List | |
| def clean_text(text: str) -> str: | |
| """ | |
| Clean raw OCR output: | |
| - Normalize unicode/special characters | |
| - Collapse multiple spaces | |
| - Remove stray control characters | |
| - Normalize line endings | |
| - Trim leading/trailing whitespace per line | |
| """ | |
| if not text: | |
| return "" | |
| # Normalize line endings | |
| text = text.replace("\r\n", "\n").replace("\r", "\n") | |
| # Remove non-printable control characters (except newline and tab) | |
| text = re.sub(r"[^\S\n\t]+", " ", text) # collapse whitespace | |
| text = re.sub(r"[^\x20-\x7E\n\t\u00A0-\uFFFF]", "", text) # remove control chars | |
| # Clean each line | |
| lines = [] | |
| for line in text.split("\n"): | |
| line = line.strip() | |
| # Remove lines that are just punctuation/symbols (OCR noise) | |
| if line and not re.fullmatch(r"[^a-zA-Z0-9\u00C0-\uFFFF]{1,3}", line): | |
| lines.append(line) | |
| # Collapse 3+ blank lines into 2 | |
| result = "\n".join(lines) | |
| result = re.sub(r"\n{3,}", "\n\n", result) | |
| return result.strip() | |
| def summarize(text: str) -> str: | |
| """ | |
| Simple extractive summary: return first 1–2 meaningful sentences. | |
| Splits on sentence-ending punctuation. | |
| """ | |
| if not text or not text.strip(): | |
| return "" | |
| # Flatten to single paragraph for sentence splitting | |
| flat = text.replace("\n", " ") | |
| flat = re.sub(r"\s+", " ", flat).strip() | |
| # Split on sentence boundaries | |
| sentences = re.split(r"(?<=[.!?])\s+", flat) | |
| sentences = [s.strip() for s in sentences if len(s.strip()) > 10] | |
| if not sentences: | |
| # Fallback: return first 150 characters | |
| return flat[:150].rstrip() + ("…" if len(flat) > 150 else "") | |
| # Return up to 2 sentences, but cap at ~200 chars | |
| summary_parts = [] | |
| char_count = 0 | |
| for sent in sentences[:2]: | |
| if char_count + len(sent) > 250 and summary_parts: | |
| break | |
| summary_parts.append(sent) | |
| char_count += len(sent) | |
| return " ".join(summary_parts) | |
| def detect_headings(text: str) -> List[str]: | |
| """ | |
| Detect heading-like lines from OCR text. | |
| A line is considered a heading if: | |
| - It is fully uppercase (and at least 3 characters) | |
| - OR it is short (≤ 6 words) and ends without terminal punctuation | |
| and starts with a capital letter | |
| """ | |
| if not text: | |
| return [] | |
| headings = [] | |
| seen = set() | |
| for line in text.split("\n"): | |
| line = line.strip() | |
| if not line or len(line) < 3: | |
| continue | |
| words = line.split() | |
| is_heading = False | |
| # Rule 1: All uppercase line | |
| if line.isupper() and len(line) >= 3: | |
| is_heading = True | |
| # Rule 2: Short title-like line (≤6 words, starts with capital, no terminal punct) | |
| elif ( | |
| 1 <= len(words) <= 6 | |
| and line[0].isupper() | |
| and not line[-1] in ".,:;!?" | |
| and not line[-1].isdigit() | |
| ): | |
| is_heading = True | |
| if is_heading: | |
| key = line.lower() | |
| if key not in seen: | |
| headings.append(line) | |
| seen.add(key) | |
| return headings | |