Spaces:
Running on Zero
Running on Zero
Download scripts/build_corpus.py from Kagzy/ResearchOS: direct link, hf CLI and curl.
- Browser
- Download file 4.14 kB
-
https://huggingface.co/spaces/Kagzy/ResearchOS/resolve/main/scripts/build_corpus.py
- Command line
-
hf download hf://spaces/Kagzy/ResearchOS/scripts/build_corpus.py
-
curl -L -o build_corpus.py https://huggingface.co/spaces/Kagzy/ResearchOS/resolve/main/scripts/build_corpus.py
4.14 kB
| """Merge harvested abstracts into the corpus: v1 full text + bulk abstracts. | |
| python scripts/build_corpus.py | |
| The v1 corpus is 8,330 papers with full text (40k passages). The harvest | |
| adds hundreds of thousands of abstracts. Both kinds go in one index: | |
| * a paper already in v1 keeps its full-text passages - the harvested | |
| abstract is dropped, so the same paper never appears twice | |
| * a new paper contributes one passage, its abstract - chunked by the same | |
| chunk_paper() as v1, so passage format and ids are identical | |
| * duplicate harvest records (a cross-listed paper seen in two sets, or a | |
| page re-fetched after an interrupted run) collapse to one | |
| The output is written beside the v1 files, never over them, so the switch is | |
| a config change and rolling back is the same change in reverse. | |
| """ | |
| from __future__ import annotations | |
| import argparse | |
| import json | |
| import sys | |
| from dataclasses import asdict | |
| from pathlib import Path | |
| sys.path.insert(0, str(Path(__file__).resolve().parents[1])) | |
| if hasattr(sys.stdout, "reconfigure"): | |
| sys.stdout.reconfigure(encoding="utf-8", errors="replace") | |
| from src.data.arxiv_source import Paper # noqa: E402 | |
| from src.data.chunking import chunk_paper # noqa: E402 | |
| def main() -> None: | |
| ap = argparse.ArgumentParser(description=__doc__) | |
| ap.add_argument("--base-papers", default="data/arxiv_papers.jsonl") | |
| ap.add_argument("--base-chunks", default="data/arxiv_chunks.jsonl") | |
| ap.add_argument("--abstracts", default="data/arxiv_abstracts.jsonl") | |
| ap.add_argument("--out-dir", default="data/corpus_v3") | |
| ap.add_argument("--min-words", type=int, default=25) | |
| args = ap.parse_args() | |
| out = Path(args.out_dir) | |
| out.mkdir(parents=True, exist_ok=True) | |
| base_ids: set[str] = set() | |
| with open(args.base_papers, encoding="utf-8") as src, \ | |
| open(out / "papers.jsonl", "w", encoding="utf-8") as dst: | |
| for line in src: | |
| if line.strip(): | |
| base_ids.add(json.loads(line)["paper_id"]) | |
| dst.write(line if line.endswith("\n") else line + "\n") | |
| harvested: dict[str, Paper] = {} | |
| bad = 0 | |
| with open(args.abstracts, encoding="utf-8") as src: | |
| for line in src: | |
| try: | |
| p = Paper(**json.loads(line)) | |
| except (json.JSONDecodeError, TypeError): | |
| bad += 1 # a line torn by an interrupted harvest | |
| continue | |
| if p.paper_id not in base_ids: | |
| harvested[p.paper_id] = p # later record wins | |
| new_papers = sorted(harvested.values(), key=lambda p: p.paper_id) | |
| n_new_chunks = 0 | |
| with open(out / "papers.jsonl", "a", encoding="utf-8") as papers_out, \ | |
| open(out / "chunks.jsonl", "w", encoding="utf-8") as chunks_out: | |
| with open(args.base_chunks, encoding="utf-8") as src: | |
| for line in src: | |
| if line.strip(): | |
| chunks_out.write(line if line.endswith("\n") else line + "\n") | |
| for p in new_papers: | |
| chunks = chunk_paper(p, min_words=args.min_words) | |
| if not chunks: | |
| continue # abstract too short to index | |
| papers_out.write(json.dumps(asdict(p), ensure_ascii=False) + "\n") | |
| for c in chunks: | |
| chunks_out.write(json.dumps(asdict(c), ensure_ascii=False) + "\n") | |
| n_new_chunks += len(chunks) | |
| by_cat: dict[str, int] = {} | |
| for p in new_papers: | |
| by_cat[p.primary_category] = by_cat.get(p.primary_category, 0) + 1 | |
| report = { | |
| "base_papers": len(base_ids), | |
| "harvested_unique_new": len(new_papers), | |
| "harvest_lines_unreadable": bad, | |
| "new_passages": n_new_chunks, | |
| "new_by_category": dict(sorted(by_cat.items(), key=lambda kv: -kv[1])), | |
| "date_range": [new_papers[0].published if new_papers else None, | |
| max((p.published for p in new_papers), default=None)], | |
| } | |
| (out / "build_report.json").write_text(json.dumps(report, indent=2), encoding="utf-8") | |
| print(json.dumps(report, indent=2)) | |
| if __name__ == "__main__": | |
| main() | |