Spaces:
Sleeping
Sleeping
File size: 4,785 Bytes
51b3964 f1cc362 51b3964 f1cc362 51b3964 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 | """
PDF helpers for the case study pipeline:
- extract_text_from_pdf / extract_source_documents: pull text out of source PDFs
(and plain text/markdown files) for a case study.
- text_to_pdf: render a generated narrative document (markdown-ish text) to PDF.
"""
from __future__ import annotations
import re
from pathlib import Path
import pdfplumber
from fpdf import FPDF
SOURCE_EXTENSIONS = {".pdf", ".txt", ".md"}
SOURCE_METADATA_FILENAMES = {
"dois.txt",
"paper_discovery_report.md",
"sources.txt",
}
# Roughly maps "smart" unicode punctuation (and a few common symbols) to ASCII so
# the core PDF fonts (latin-1 only) can render them without crashing.
_UNICODE_REPLACEMENTS = {
"β": "'", "β": "'", # single quotes
"β": '"', "β": '"', # double quotes
"β": "-", "β": "-", # en/em dash
"β¦": "...", # ellipsis
"Β ": " ", # non-breaking space
"β’": "-", # bullet
"β": "->",
"β": "<-",
"Γ": "x",
}
def _sanitize(text: str) -> str:
for src, dst in _UNICODE_REPLACEMENTS.items():
text = text.replace(src, dst)
# Drop any remaining characters the core fonts can't encode.
return text.encode("latin-1", "replace").decode("latin-1")
def extract_text_from_pdf(path: Path) -> str:
"""Extract all text from a PDF, page by page."""
chunks = []
with pdfplumber.open(path) as pdf:
for i, page in enumerate(pdf.pages, start=1):
page_text = page.extract_text() or ""
chunks.append(f"--- Page {i} ---\n{page_text}")
return "\n\n".join(chunks)
def extract_source_documents(folder: Path, max_chars_per_file: int | None = None) -> str:
"""
Read every PDF / .txt / .md file directly inside `folder` and return a single
string with each file's text labelled by filename, ready to drop into a prompt.
`max_chars_per_file` truncates very long individual files (set to None for no
truncation).
"""
folder = Path(folder)
if not folder.is_dir():
raise FileNotFoundError(f"Source documents folder not found: {folder}")
files = sorted(
p for p in folder.iterdir()
if (
p.is_file()
and p.suffix.lower() in SOURCE_EXTENSIONS
and p.name not in SOURCE_METADATA_FILENAMES
)
)
if not files:
raise FileNotFoundError(
f"No source documents (.pdf, .txt, .md) found in {folder}"
)
sections = []
for f in files:
if f.suffix.lower() == ".pdf":
text = extract_text_from_pdf(f)
else:
text = f.read_text(encoding="utf-8", errors="replace")
if max_chars_per_file and len(text) > max_chars_per_file:
text = text[:max_chars_per_file] + "\n\n[... truncated ...]"
sections.append(f"### Source file: {f.name} ###\n{text}")
return "\n\n".join(sections)
def text_to_pdf(text: str, output_path: Path, title: str | None = None) -> Path:
"""
Render a generated narrative document (plain text with light markdown-style
`#`/`##`/`###` headings) to a PDF at `output_path`.
"""
output_path = Path(output_path)
pdf = FPDF(format="Letter")
pdf.set_auto_page_break(auto=True, margin=15)
pdf.add_page()
def write_cell(h: float, content: str) -> None:
"""multi_cell, then reset cursor to the left margin (fpdf2 leaves x at the
right edge after a width=0 multi_cell, which breaks the next call)."""
pdf.multi_cell(0, h, content)
pdf.set_x(pdf.l_margin)
if title:
pdf.set_font("Helvetica", style="B", size=16)
write_cell(10, _sanitize(title))
pdf.ln(4)
for raw_line in text.splitlines():
line = raw_line.rstrip()
if not line.strip():
pdf.ln(3)
continue
heading_match = re.match(r"^(#{1,4})\s+(.*)", line)
if heading_match:
level = len(heading_match.group(1))
content = heading_match.group(2).strip()
size = {1: 15, 2: 13, 3: 12, 4: 11}.get(level, 11)
pdf.set_font("Helvetica", style="B", size=size)
pdf.ln(2)
write_cell(8, _sanitize(content))
pdf.set_font("Helvetica", size=10)
continue
# Bold "Question:" / "Answer:" / section markers written without '#'
if re.match(r"^(Question|Answer|Narrative)\s*:", line, re.IGNORECASE):
pdf.set_font("Helvetica", style="B", size=10)
write_cell(6, _sanitize(line))
pdf.set_font("Helvetica", size=10)
continue
pdf.set_font("Helvetica", size=10)
write_cell(6, _sanitize(line))
output_path.parent.mkdir(parents=True, exist_ok=True)
pdf.output(str(output_path))
return output_path
|