Spaces:
Sleeping
Sleeping
File size: 2,239 Bytes
1fee1c2 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 | # COST: OpenAI text-embedding-3-small called during indexing (one-time per document).
# No chat LLM is invoked. Embedding cost is ~$0.00002/1K tokens — negligible for
# typical documents. Upsert semantics mean re-indexing the same file is idempotent
# but does re-embed (and re-bill) all chunks.
import sys
from pathlib import Path
sys.path.insert(0, str(Path(__file__).parent.parent))
from dotenv import load_dotenv
load_dotenv()
from ingestion import chunk_text, embed_and_store, parse_document
def load_and_index_document(file_path: str) -> str:
"""Parse, chunk, embed, and store a single document into the vector store.
Accepts a .pdf or .docx file, extracts text (preserving per-page structure
for PDFs), splits it into overlapping 500-word chunks with a 50-word overlap,
and upserts every chunk with source metadata into the persistent ChromaDB
collection. Calling this function on an already-indexed file is safe — all
chunks are upserted so duplicates are automatically overwritten.
Args:
file_path: Absolute or relative path to the document to ingest.
Must be a .pdf or .docx file.
Returns:
A human-readable confirmation string such as:
"Successfully indexed 'report.pdf': 42 chunks stored across 10 page(s)."
If the file cannot be parsed an error message is returned instead of
raising, so the agent can surface it gracefully.
"""
path = Path(file_path)
if not path.exists():
return f"Error: file not found at '{file_path}'."
try:
pages = parse_document(path)
except ValueError as exc:
return f"Error: {exc}"
total_chunks = 0
for page in pages:
chunks = chunk_text(page["text"])
embed_and_store(
chunks,
metadata={
"source_file": path.name,
"page_number": page["page_number"],
},
)
total_chunks += len(chunks)
page_count = len(pages)
page_label = f"{page_count} page(s)" if pages[0]["page_number"] is not None else "1 section"
return (
f"Successfully indexed '{path.name}': "
f"{total_chunks} chunks stored across {page_label}."
)
|