Upload main.py

main.py

@@ -46,13 +46,15 @@ def extract_text_from_pdf(pdf_path: str) -> str:
         raise HTTPException(status_code=400, detail="PDF contains no extractable text.")
     return text
 
-def chunk_text(text: str, max_tokens: int = 400) -> list:
-    """Chunks text into manageable pieces."""
-    tokens = tokenizer.encode(text)
-    return [
-        tokenizer.decode(tokens[i : i + max_tokens])
+def chunk_text(text: str, max_tokens: int = 512) -> list:
+    """Chunks text into pieces within the model's token limit."""
+    tokens = tokenizer.encode(text, truncation=False)  # No truncation yet
+    chunks = [
+        tokens[i: i + max_tokens]
         for i in range(0, len(tokens), max_tokens)
     ]
+    return [tokenizer.decode(chunk) for chunk in chunks]
+
 
 def generate_chunk_output(chunk: str) -> dict:
     """Processes a chunk of text with flan-t5-base."""