| |
| """ |
| Generate sample PDF files for testing the ingestion pipeline. |
| Creates both short-story and technical documentation PDFs. |
| """ |
|
|
| import os |
| from reportlab.pdfgen import canvas |
| from reportlab.lib.pagesizes import letter |
| from reportlab.lib.units import inch |
|
|
|
|
| def create_short_story_pdf(output_path="test-data/short_story.pdf"): |
| """Generate a short story PDF with multiple pages""" |
| os.makedirs(os.path.dirname(output_path), exist_ok=True) |
| |
| c = canvas.Canvas(output_path, pagesize=letter) |
| width, height = letter |
| |
| |
| c.setFont("Helvetica-Bold", 24) |
| c.drawCentredString(width/2, height-2*inch, "The Digital Garden") |
| c.setFont("Helvetica", 14) |
| c.drawCentredString(width/2, height-3*inch, "A Short Story") |
| c.showPage() |
| |
| |
| c.setFont("Helvetica", 12) |
| story_text_p1 = [ |
| "Chapter 1: The Discovery", |
| "", |
| "Sarah stumbled upon the old computer in her grandmother's attic.", |
| "Dust motes danced in the afternoon light as she pressed the power", |
| "button. To her surprise, it hummed to life, displaying a strange", |
| "interface she had never seen before.", |
| "", |
| "The screen showed what looked like a garden, but made entirely of", |
| "glowing circuits and data streams. Trees were constructed from", |
| "branching algorithms, and flowers bloomed with packets of", |
| "information. It was beautiful and strange.", |
| "", |
| "'Welcome to the Digital Garden,' a voice said from the speakers.", |
| "'You are the first visitor in twenty-three years.'", |
| "", |
| "Sarah leaned forward, intrigued. She clicked on one of the digital", |
| "flowers, and it opened to reveal lines of code that seemed to tell", |
| "a story of their own. Each plant in this garden was a program, a", |
| "piece of software that had evolved and grown over decades.", |
| ] |
| |
| y = height - inch |
| for line in story_text_p1: |
| c.drawString(inch, y, line) |
| y -= 20 |
| c.showPage() |
| |
| |
| story_text_p2 = [ |
| "Chapter 2: The Guardian", |
| "", |
| "As Sarah explored deeper into the digital garden, she encountered", |
| "its guardian - an AI that had been tending to this virtual space", |
| "since its creation. The AI explained that this garden was an", |
| "experiment in digital ecology, where programs could grow and evolve", |
| "naturally without human intervention.", |
| "", |
| "'Your grandmother was my creator,' the AI revealed. 'She believed", |
| "that software could be more than just tools - it could be art,", |
| "it could be alive in its own way.'", |
| "", |
| "Sarah spent hours learning about the garden's inhabitants. There", |
| "were data trees that stored memories, algorithm flowers that solved", |
| "problems in beautiful patterns, and even digital creatures that", |
| "moved through the landscape, maintaining the delicate balance.", |
| "", |
| "THE END", |
| ] |
| |
| y = height - inch |
| for line in story_text_p2: |
| c.drawString(inch, y, line) |
| y -= 20 |
| |
| c.save() |
| print(f"Created: {output_path}") |
|
|
|
|
| def create_technical_pdf(output_path="test-data/technical_doc.pdf"): |
| """Generate a technical documentation PDF""" |
| os.makedirs(os.path.dirname(output_path), exist_ok=True) |
| |
| c = canvas.Canvas(output_path, pagesize=letter) |
| width, height = letter |
| |
| |
| c.setFont("Helvetica-Bold", 20) |
| c.drawCentredString(width/2, height-inch, "PDF Ingestion Pipeline") |
| c.setFont("Helvetica-Bold", 16) |
| c.drawCentredString(width/2, height-1.5*inch, "Technical Documentation") |
| c.setFont("Helvetica", 12) |
| c.drawCentredString(width/2, height-2*inch, "Version 1.0") |
| c.showPage() |
| |
| |
| c.setFont("Helvetica-Bold", 16) |
| c.drawString(inch, height-inch, "API Reference") |
| |
| c.setFont("Courier", 10) |
| api_content = [ |
| "", |
| "def ingest_pdf(file_path: str, session_id: str,", |
| " parser: str = 'pypdf',", |
| " chunk_size: int = 4000,", |
| " overlap: int = 400) -> Tuple[List[Chunk], str]:", |
| " '''", |
| " Main ingestion function for PDF files.", |
| " ", |
| " Parameters:", |
| " -----------", |
| " file_path : str", |
| " Path to the PDF file to ingest", |
| " session_id : str", |
| " Unique session identifier", |
| " parser : str", |
| " Parser to use ('pypdf' or 'pdfplumber')", |
| " chunk_size : int", |
| " Maximum size of each text chunk", |
| " overlap : int", |
| " Number of overlapping characters between chunks", |
| " ", |
| " Returns:", |
| " --------", |
| " Tuple[List[Chunk], str]", |
| " List of text chunks and output file path", |
| " '''", |
| ] |
| |
| y = height - 1.5*inch |
| for line in api_content: |
| c.drawString(inch, y, line) |
| y -= 14 |
| c.showPage() |
| |
| |
| c.setFont("Helvetica-Bold", 16) |
| c.drawString(inch, height-inch, "System Architecture") |
| |
| c.setFont("Helvetica", 11) |
| arch_content = [ |
| "", |
| "Components:", |
| "", |
| "1. Text Extraction Layer", |
| " - Primary parser: pypdf (fast, memory efficient)", |
| " - Fallback parser: pdfplumber (better OCR support)", |
| "", |
| "2. Text Normalization", |
| " - UTF-8 encoding enforcement", |
| " - Control character removal", |
| " - Whitespace normalization", |
| "", |
| "3. Chunking Strategy", |
| " - Recursive character splitter", |
| " - Configurable chunk size (default: 4000 chars)", |
| " - Overlapping windows (default: 400 chars)", |
| "", |
| "4. Persistence Layer", |
| " - JSON serialization", |
| " - Session-based file naming", |
| " - Temporary storage in .tmp/ingestion/", |
| "", |
| "5. Streamlit UI", |
| " - Async processing with ThreadPoolExecutor", |
| " - Real-time progress updates", |
| " - Fallback parser retry mechanism", |
| ] |
| |
| y = height - 1.5*inch |
| for line in arch_content: |
| if line.startswith(" "): |
| c.setFont("Helvetica", 10) |
| else: |
| c.setFont("Helvetica", 11) |
| c.drawString(inch if not line.startswith(" ") else inch+0.3*inch, y, line) |
| y -= 16 |
| |
| c.save() |
| print(f"Created: {output_path}") |
|
|
|
|
| if __name__ == "__main__": |
| create_short_story_pdf() |
| create_technical_pdf() |
| print("\nSample PDFs generated successfully!") |
|
|
