from __future__ import annotations import hashlib from typing import Any from pdf_utils import extract_pdf_text_by_page from chunking import chunk_text def _stable_id(*parts: str) -> str: raw = "||".join(parts).encode("utf-8", errors="ignore") return hashlib.sha1(raw).hexdigest() def build_vectors_for_pdf( pdf_bytes: bytes, filename: str, embed_fn, chunk_size: int, overlap: int, ) -> list[dict[str, Any]]: pages = extract_pdf_text_by_page(pdf_bytes) all_vectors: list[dict[str, Any]] = [] for p in pages: page_num = p["page"] text = p["text"] if not text: continue chunks = chunk_text(text, chunk_size=chunk_size, overlap=overlap) if not chunks: continue embeddings = embed_fn(chunks) for idx, (chunk, vec) in enumerate(zip(chunks, embeddings)): vid = _stable_id(filename, str(page_num), str(idx), chunk[:80]) metadata = { "filename": filename, "page": page_num, "chunk_index": idx, "text": chunk, } all_vectors.append({"id": vid, "values": vec, "metadata": metadata}) return all_vectors