Spaces:
Running on Zero
Running on Zero
Download src/data/loader.py from Kagzy/ResearchOS: direct link, hf CLI and curl.
- Browser
- Download file 2.08 kB
-
https://huggingface.co/spaces/Kagzy/ResearchOS/resolve/main/src/data/loader.py
- Command line
-
hf download hf://spaces/Kagzy/ResearchOS/src/data/loader.py
-
curl -L -o loader.py https://huggingface.co/spaces/Kagzy/ResearchOS/resolve/main/src/data/loader.py
2.08 kB
| """SciFact loading via BEIR, plus the corpus flattening the retrievers need.""" | |
| from __future__ import annotations | |
| from pathlib import Path | |
| # beir is deliberately NOT imported at module level. Serving imports this module | |
| # for Corpus and passage_text only, and the deployed Space does not install the | |
| # evaluation stack - a top-level import took the whole app down on startup. | |
| Corpus = dict[str, dict[str, str]] | |
| Queries = dict[str, str] | |
| Qrels = dict[str, dict[str, int]] | |
| def download_dataset(url: str, cache_dir: str = "datasets") -> str: | |
| """Download and unzip a BEIR dataset. Returns the extracted folder path.""" | |
| from beir import util | |
| name = url.rstrip("/").split("/")[-1].removesuffix(".zip") | |
| target = Path(cache_dir) / name | |
| if target.exists(): | |
| return str(target) | |
| return util.download_and_unzip(url, cache_dir) | |
| def load_split(url: str, split: str, cache_dir: str = "datasets") -> tuple[Corpus, Queries, Qrels]: | |
| """Load one split of a BEIR dataset.""" | |
| from beir.datasets.data_loader import GenericDataLoader | |
| data_path = download_dataset(url, cache_dir) | |
| return GenericDataLoader(data_folder=data_path).load(split=split) | |
| def flatten_corpus(corpus: Corpus) -> tuple[list[str], list[str]]: | |
| """Turn the corpus dict into parallel id/text lists. | |
| Index i of both lists always describes the same document. Retrievers work in | |
| positions; everything else in this project works in doc ids. | |
| """ | |
| doc_ids: list[str] = [] | |
| doc_texts: list[str] = [] | |
| for doc_id, doc in corpus.items(): | |
| doc_ids.append(doc_id) | |
| title = doc.get("title", "") or "" | |
| text = doc.get("text", "") or "" | |
| doc_texts.append(f"{title} {text}".strip()) | |
| return doc_ids, doc_texts | |
| def passage_text(corpus: Corpus, doc_id: str) -> str: | |
| """The exact text representation used for both indexing and reranking.""" | |
| doc = corpus[doc_id] | |
| title = doc.get("title", "") or "" | |
| text = doc.get("text", "") or "" | |
| return f"{title} {text}".strip() | |