Spaces:
Runtime error
Runtime error
Download src/ingest.py from ResumeScreening/resume-screening-rag: direct link, hf CLI and curl.
- Browser
- Download file 1.23 kB
-
https://huggingface.co/spaces/ResumeScreening/resume-screening-rag/resolve/main/src/ingest.py
- Command line
-
hf download hf://spaces/ResumeScreening/resume-screening-rag/src/ingest.py
-
curl -L -o ingest.py https://huggingface.co/spaces/ResumeScreening/resume-screening-rag/resolve/main/src/ingest.py
1.23 kB
| from __future__ import annotations | |
| import hashlib | |
| from typing import Any | |
| from pdf_utils import extract_pdf_text_by_page | |
| from chunking import chunk_text | |
| def _stable_id(*parts: str) -> str: | |
| raw = "||".join(parts).encode("utf-8", errors="ignore") | |
| return hashlib.sha1(raw).hexdigest() | |
| def build_vectors_for_pdf( | |
| pdf_bytes: bytes, | |
| filename: str, | |
| embed_fn, | |
| chunk_size: int, | |
| overlap: int, | |
| ) -> list[dict[str, Any]]: | |
| pages = extract_pdf_text_by_page(pdf_bytes) | |
| all_vectors: list[dict[str, Any]] = [] | |
| for p in pages: | |
| page_num = p["page"] | |
| text = p["text"] | |
| if not text: | |
| continue | |
| chunks = chunk_text(text, chunk_size=chunk_size, overlap=overlap) | |
| if not chunks: | |
| continue | |
| embeddings = embed_fn(chunks) | |
| for idx, (chunk, vec) in enumerate(zip(chunks, embeddings)): | |
| vid = _stable_id(filename, str(page_num), str(idx), chunk[:80]) | |
| metadata = { | |
| "filename": filename, | |
| "page": page_num, | |
| "chunk_index": idx, | |
| "text": chunk, | |
| } | |
| all_vectors.append({"id": vid, "values": vec, "metadata": metadata}) | |
| return all_vectors |