File size: 4,785 Bytes
51b3964
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
f1cc362
 
 
 
 
51b3964
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
f1cc362
 
 
 
 
51b3964
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
"""
PDF helpers for the case study pipeline:
- extract_text_from_pdf / extract_source_documents: pull text out of source PDFs
  (and plain text/markdown files) for a case study.
- text_to_pdf: render a generated narrative document (markdown-ish text) to PDF.
"""

from __future__ import annotations

import re
from pathlib import Path

import pdfplumber
from fpdf import FPDF

SOURCE_EXTENSIONS = {".pdf", ".txt", ".md"}
SOURCE_METADATA_FILENAMES = {
    "dois.txt",
    "paper_discovery_report.md",
    "sources.txt",
}

# Roughly maps "smart" unicode punctuation (and a few common symbols) to ASCII so
# the core PDF fonts (latin-1 only) can render them without crashing.
_UNICODE_REPLACEMENTS = {
    "β€˜": "'", "’": "'",  # single quotes
    "β€œ": '"', "”": '"',  # double quotes
    "–": "-", "β€”": "-",  # en/em dash
    "…": "...",  # ellipsis
    "Β ": " ",  # non-breaking space
    "β€’": "-",  # bullet
    "β†’": "->",
    "←": "<-",
    "Γ—": "x",
}


def _sanitize(text: str) -> str:
    for src, dst in _UNICODE_REPLACEMENTS.items():
        text = text.replace(src, dst)
    # Drop any remaining characters the core fonts can't encode.
    return text.encode("latin-1", "replace").decode("latin-1")


def extract_text_from_pdf(path: Path) -> str:
    """Extract all text from a PDF, page by page."""
    chunks = []
    with pdfplumber.open(path) as pdf:
        for i, page in enumerate(pdf.pages, start=1):
            page_text = page.extract_text() or ""
            chunks.append(f"--- Page {i} ---\n{page_text}")
    return "\n\n".join(chunks)


def extract_source_documents(folder: Path, max_chars_per_file: int | None = None) -> str:
    """
    Read every PDF / .txt / .md file directly inside `folder` and return a single
    string with each file's text labelled by filename, ready to drop into a prompt.

    `max_chars_per_file` truncates very long individual files (set to None for no
    truncation).
    """
    folder = Path(folder)
    if not folder.is_dir():
        raise FileNotFoundError(f"Source documents folder not found: {folder}")

    files = sorted(
        p for p in folder.iterdir()
        if (
            p.is_file()
            and p.suffix.lower() in SOURCE_EXTENSIONS
            and p.name not in SOURCE_METADATA_FILENAMES
        )
    )
    if not files:
        raise FileNotFoundError(
            f"No source documents (.pdf, .txt, .md) found in {folder}"
        )

    sections = []
    for f in files:
        if f.suffix.lower() == ".pdf":
            text = extract_text_from_pdf(f)
        else:
            text = f.read_text(encoding="utf-8", errors="replace")

        if max_chars_per_file and len(text) > max_chars_per_file:
            text = text[:max_chars_per_file] + "\n\n[... truncated ...]"

        sections.append(f"### Source file: {f.name} ###\n{text}")

    return "\n\n".join(sections)


def text_to_pdf(text: str, output_path: Path, title: str | None = None) -> Path:
    """
    Render a generated narrative document (plain text with light markdown-style
    `#`/`##`/`###` headings) to a PDF at `output_path`.
    """
    output_path = Path(output_path)
    pdf = FPDF(format="Letter")
    pdf.set_auto_page_break(auto=True, margin=15)
    pdf.add_page()

    def write_cell(h: float, content: str) -> None:
        """multi_cell, then reset cursor to the left margin (fpdf2 leaves x at the
        right edge after a width=0 multi_cell, which breaks the next call)."""
        pdf.multi_cell(0, h, content)
        pdf.set_x(pdf.l_margin)

    if title:
        pdf.set_font("Helvetica", style="B", size=16)
        write_cell(10, _sanitize(title))
        pdf.ln(4)

    for raw_line in text.splitlines():
        line = raw_line.rstrip()

        if not line.strip():
            pdf.ln(3)
            continue

        heading_match = re.match(r"^(#{1,4})\s+(.*)", line)
        if heading_match:
            level = len(heading_match.group(1))
            content = heading_match.group(2).strip()
            size = {1: 15, 2: 13, 3: 12, 4: 11}.get(level, 11)
            pdf.set_font("Helvetica", style="B", size=size)
            pdf.ln(2)
            write_cell(8, _sanitize(content))
            pdf.set_font("Helvetica", size=10)
            continue

        # Bold "Question:" / "Answer:" / section markers written without '#'
        if re.match(r"^(Question|Answer|Narrative)\s*:", line, re.IGNORECASE):
            pdf.set_font("Helvetica", style="B", size=10)
            write_cell(6, _sanitize(line))
            pdf.set_font("Helvetica", size=10)
            continue

        pdf.set_font("Helvetica", size=10)
        write_cell(6, _sanitize(line))

    output_path.parent.mkdir(parents=True, exist_ok=True)
    pdf.output(str(output_path))
    return output_path