btran299's picture
Update src/ingest.py
447f0e2 verified
Raw History Blame Contribute Delete
1.23 kB
from __future__ import annotations
import hashlib
from typing import Any
from pdf_utils import extract_pdf_text_by_page
from chunking import chunk_text
def _stable_id(*parts: str) -> str:
raw = "||".join(parts).encode("utf-8", errors="ignore")
return hashlib.sha1(raw).hexdigest()
def build_vectors_for_pdf(
pdf_bytes: bytes,
filename: str,
embed_fn,
chunk_size: int,
overlap: int,
) -> list[dict[str, Any]]:
pages = extract_pdf_text_by_page(pdf_bytes)
all_vectors: list[dict[str, Any]] = []
for p in pages:
page_num = p["page"]
text = p["text"]
if not text:
continue
chunks = chunk_text(text, chunk_size=chunk_size, overlap=overlap)
if not chunks:
continue
embeddings = embed_fn(chunks)
for idx, (chunk, vec) in enumerate(zip(chunks, embeddings)):
vid = _stable_id(filename, str(page_num), str(idx), chunk[:80])
metadata = {
"filename": filename,
"page": page_num,
"chunk_index": idx,
"text": chunk,
}
all_vectors.append({"id": vid, "values": vec, "metadata": metadata})
return all_vectors