Merge pull request #1 from francisco-magana/franciscomaganacontact/nah-17-extract_text-pdf-pastedtxt-length-based-chunker

app.py

@@ -24,16 +24,39 @@ def start_session(file, pasted_text):
     text = pasted_text.strip() if pasted_text and pasted_text.strip() else cp.extract_text(file)
     if not text:
         return (None, gr.update(visible=True), gr.update(visible=False),
-                "⚠️ No text found. Upload a PDF or paste some notes.", "", "")
-    deck = cp.generate_deck(text)
+                "⚠️ No text found. Upload a PDF or paste some notes.", "", "",
+                gr.update(visible=False), None)
+    chunks = cp._chunk(text)
+    debug = {
+        "total_chars": len(text),
+        "extracted_text_preview": text[:800] + ("…" if len(text) > 800 else ""),
+        "chunk_count": len(chunks),
+        "chunks": [
+            {
+                "index": i,
+                "chars": len(c),
+                "tokens_approx": len(c) // 4,
+                "text": c,
+            }
+            for i, c in enumerate(chunks)
+        ],
+    }
+    try:
+        deck = cp.generate_deck(text)
+    except Exception as e:
+        return (None, gr.update(visible=True), gr.update(visible=False),
+                f"⚠️ generate_deck failed ({type(e).__name__}: {e}). "
+                "Extraction & chunks are shown below.",
+                "", "", gr.update(visible=True), debug)
     if not deck:
         return (None, gr.update(visible=True), gr.update(visible=False),
                 "⚠️ Couldn't generate questions from that. Try different material.",
-                "", "")
+                "", "", gr.update(visible=True), debug)
     session = le.init_session(deck)
     card = le.next_card(session)
     return (session, gr.update(visible=False), gr.update(visible=True),
-            "", _progress(session), card["question"] if card else "")
+            "", _progress(session), card["question"] if card else "",
+            gr.update(visible=True), debug)
 
 
 def submit_answer(session, user_answer):
@@ -111,9 +134,14 @@ with gr.Blocks(title="Recall — your AI study partner", theme=gr.themes.Soft())
     with gr.Group(visible=False) as recap_view:
         recap_md = gr.Markdown("")
 
+    # Debug panel — visible after each upload so you can verify extraction & chunks
+    with gr.Accordion("🔍 Debug: extraction & chunks", open=False, visible=False) as debug_accordion:
+        debug_data = gr.JSON(label="Content pipeline output")
+
     start_btn.click(
         start_session, [file_in, text_in],
-        [session, upload_view, study_view, upload_msg, progress, question],
+        [session, upload_view, study_view, upload_msg, progress, question,
+         debug_accordion, debug_data],
     )
     submit_btn.click(
         submit_answer, [session, answer_in],

content_pipeline.py

@@ -10,6 +10,7 @@ learning_engine depend on them.
 from __future__ import annotations
 
 import os
+import re
 
 import llm
 from schema import Card, new_card, validate_card
@@ -34,7 +35,7 @@ def extract_text(file) -> str:
         # TODO(Frank): pip install pypdf
         from pypdf import PdfReader
         reader = PdfReader(path)
-        text = "\n".join((page.extract_text() or "") for page in reader.pages)
+        text = "\n\n".join((page.extract_text() or "") for page in reader.pages)
     else:
         with open(path, "r", encoding="utf-8", errors="ignore") as f:
             text = f.read()
@@ -85,10 +86,31 @@ def _normalize(text: str) -> str:
     return " ".join(text.split())
 
 
-def _chunk(text: str, size: int = 800) -> list[str]:
-    """Simple length-based splitter. No vector DB — keep it boring."""
-    words = text.split()
-    return [" ".join(words[i:i + size]) for i in range(0, len(words), size)] or [text]
+def _chunk(text: str, size: int = 2500) -> list[str]:
+    """
+    Character-based splitter targeting ~500–800 tokens per chunk (1 token ≈ 4 chars,
+    so size=2500 ≈ 625 tokens). Breaks at sentence boundaries where possible.
+    """
+    # Split into sentences on . ! ? followed by whitespace or end-of-string.
+    sentences = re.split(r'(?<=[.!?])\s+', text.strip())
+    chunks: list[str] = []
+    current: list[str] = []
+    current_len = 0
+
+    for sentence in sentences:
+        slen = len(sentence)
+        if current and current_len + 1 + slen > size:
+            chunks.append(" ".join(current))
+            current = [sentence]
+            current_len = slen
+        else:
+            current.append(sentence)
+            current_len += (1 + slen) if current_len else slen
+
+    if current:
+        chunks.append(" ".join(current))
+
+    return chunks or [text]
 
 
 def _cards_from_chunk(chunk: str) -> list[Card]:
@@ -112,7 +134,7 @@ def _cards_from_chunk(chunk: str) -> list[Card]:
             question=str(item.get("question", "")).strip(),
             answer=str(item.get("answer", "")).strip(),
             topic=str(item.get("topic", "General")).strip() or "General",
-            source_chunk=chunk[:300],
+            source_chunk=chunk,
             difficulty=int(item.get("difficulty", 1) or 1),
         )
         if validate_card(card):