File size: 4,562 Bytes
e5c059f
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
"""Publish the corpus and its prebuilt indices to a Hugging Face dataset repo.

    python scripts/publish_corpus.py --repo Kagzy/researchos-corpus

A free Space's own repo holds 1 GB; the 273k-paper corpus with its indices is
~1.6 GB. The Space downloads this dataset on first start instead
(src/api/corpus_fetch.py). Resumable: upload_large_folder hashes and uploads
in pieces and skips what is already there, so an interrupted upload continues
rather than restarting.

The paper records are slimmed first - `full_text` (82 MB of PDF text for the
8k v1 papers) is never read at serving time; chunks.jsonl already holds it.
"""

from __future__ import annotations

import argparse
import os
import shutil
import sys
from pathlib import Path

sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
sys.path.insert(0, str(Path(__file__).resolve().parent))
if hasattr(sys.stdout, "reconfigure"):
    sys.stdout.reconfigure(encoding="utf-8", errors="replace")

from prepare_deploy import slim_papers  # noqa: E402
from src.config import load_config  # noqa: E402

CARD = """---
license: cc0-1.0
language: [en]
pretty_name: ResearchOS corpus
size_categories: [100K<n<1M]
task_categories: [text-retrieval]
tags: [arxiv, machine-learning, retrieval, rag]
---

# ResearchOS corpus

The search corpus behind [ResearchOS](https://github.com/Kagzyyy/ResearchOS): AI
papers from arXiv, plus the BM25 and dense (BAAI/bge-small-en-v1.5) indices
prebuilt over them, so a server can start in seconds instead of embedding
~{passages:,} passages.

| | |
|---|---|
| papers | {papers:,} |
| passages | {passages:,} |
| coverage | arXiv cs.LG, cs.AI, cs.CL, cs.CV, cs.RO, cs.IR, cs.NE and stat.ML, first submitted 2024-01 onward, plus {full_text:,} earlier papers with full text |
| source | arXiv OAI-PMH metadata, which arXiv releases under CC0 |

```
data/corpus_v3/papers.jsonl    one paper per line (title, abstract, authors, category, dates)
data/corpus_v3/chunks.jsonl    retrievable passages (an abstract, or a section window of a full text)
cache/ml_v3/bm25/              bm25s index   (corpus.json = fingerprint of the exact passage ids)
cache/ml_v3/dense_*.npy        float32 embeddings, row i = passage i of chunks.jsonl
```

The indices are valid only for these exact files: each is fingerprinted to the
ordered passage ids, and the loader rebuilds rather than trust a mismatch.
Rebuild with `scripts/harvest_arxiv_oai.py`, `scripts/build_corpus.py` and
`scripts/build_index.py`.
"""


def main() -> None:
    ap = argparse.ArgumentParser(description=__doc__)
    ap.add_argument("--repo", default="Kagzy/researchos-corpus")
    ap.add_argument("--config", default="configs/ml.yaml")
    ap.add_argument("--stage", default=None, help="staging dir (default: beside the corpus)")
    ap.add_argument("--private", action="store_true")
    args = ap.parse_args()

    paths = load_config(args.config)["paths"]
    papers, chunks, cache = Path(paths["papers"]), Path(paths["chunks"]), Path(paths["index_cache"])
    stage = Path(args.stage or papers.parent.parent / "_publish_stage")
    if stage.exists():
        shutil.rmtree(stage)
    (stage / papers.parent).mkdir(parents=True)

    print("[publish] slimming papers ...", flush=True)
    n_papers, before, after = slim_papers(papers, stage / papers)
    print(f"[publish] {n_papers:,} papers, {before / 1e6:.0f} -> {after / 1e6:.0f} MB", flush=True)

    def link(src: Path, dst: Path) -> None:
        dst.parent.mkdir(parents=True, exist_ok=True)
        try:
            os.link(src, dst)           # same volume: no extra disk
        except OSError:
            shutil.copy2(src, dst)

    link(chunks, stage / chunks)
    for f in cache.rglob("*"):
        if f.is_file():
            link(f, stage / f)

    n_passages = sum(1 for _ in open(chunks, encoding="utf-8"))
    n_full = sum(1 for line in open(papers, encoding="utf-8")
                 if '"full_text": "' in line)
    (stage / "README.md").write_text(
        CARD.format(papers=n_papers, passages=n_passages, full_text=n_full), encoding="utf-8")

    from huggingface_hub import HfApi

    api = HfApi()
    api.create_repo(args.repo, repo_type="dataset", private=args.private, exist_ok=True)
    print(f"[publish] uploading {stage} -> {args.repo} (resumable) ...", flush=True)
    api.upload_large_folder(repo_id=args.repo, repo_type="dataset", folder_path=str(stage))
    print(f"[publish] done: https://huggingface.co/datasets/{args.repo}", flush=True)
    shutil.rmtree(stage, ignore_errors=True)


if __name__ == "__main__":
    main()