day1demo / utils /file_parser.py
sg-altis's picture
v0 claim extraction and various code health improvements
8cb9c5c
Raw History Blame Contribute Delete
2.82 kB
import os
import fitz # PyMuPDF
from pypdf import PdfReader
from models.internal import Document, SourceType
def infer_file_type(file_path: str) -> SourceType:
"""
Infer the SourceType from the filename/path.
This is a simple heuristic based on common naming patterns.
"""
file_lower = file_path.lower()
if "board" in file_lower or "deck" in file_lower:
return SourceType.BOARD_DECK
elif "dataroom" in file_lower or "data room" in file_lower:
return SourceType.DATAROOM
elif "transcript" in file_lower or "call" in file_lower or "tegus" in file_lower:
return SourceType.CALL_TRANSCRIPT
elif "research" in file_lower or "deep dive" in file_lower or "analysis" in file_lower:
return SourceType.PRIMARY_RESEARCH
elif any(
kw in file_lower
for kw in [
"pitchbook",
"alphasense",
"crunchbase",
"cb insights",
"dealroom",
"tracxn",
"harmonic",
"owler",
"zoominfo",
"company profile",
"company overview",
"export",
"download",
]
):
return SourceType.COMPANY_PROFILE
elif any(kw in file_lower for kw in ["gartner", "forrester", "idc", "mckinsey", "bain"]):
return SourceType.VENDOR_REPORT
elif "note" in file_lower or "memo" in file_lower:
return SourceType.NOTE
else:
return SourceType.UNDEFINED
def pdf_to_text_pymupdf(path: str) -> str:
"""
Extract plain text from a PDF at `path` and return it as one string.
Works for text-based PDFs (not scanned images unless OCR is used).
"""
doc = fitz.open(path)
try:
parts: list[str] = []
for page in doc:
parts.append(page.get_text("text") or "")
return "\n\n".join(parts).strip()
finally:
doc.close()
def get_upload_document(file_path: str, parser: str = "pymupdf") -> Document:
"""
Create a Document from an uploaded file path.
Supports PDF and text files.
"""
file_name = os.path.basename(file_path)
file_source = infer_file_type(file_path)
if file_path.lower().endswith(".pdf"):
if parser == "pymupdf":
file_text = pdf_to_text_pymupdf(file_path)
else:
# Fallback to pypdf if needed
reader = PdfReader(file_path)
file_text = ""
for page in reader.pages:
file_text += (page.extract_text() or "") + "\n"
return Document(name=file_name, source_type=file_source, content=file_text)
else:
with open(file_path, encoding="utf-8") as file:
file_text = file.read()
return Document(name=file_name, source_type=file_source, content=file_text)