#!/usr/bin/env python """Build the real training corpus into a memory-mapped token cache. python scripts/build_corpus.py [--manifest data/manifest.json] [--out DIR] Fetches every enabled source in the manifest, strips it to prose, deduplicates, checks it against the held-out eval set, and writes one uint16 token file that training memory-maps. The download is tens of megabytes; the resulting cache is not committed. """ from __future__ import annotations import argparse import json import sys from pathlib import Path sys.path.insert(0, str(Path(__file__).resolve().parents[1])) from algo.corpus import CACHE_DIR, build_corpus # noqa: E402 from algo.data import MANIFEST_PATH, SimpleTokenizer # noqa: E402 def main() -> int: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--manifest", default=str(MANIFEST_PATH)) parser.add_argument("--out", default=str(CACHE_DIR)) args = parser.parse_args() metadata = build_corpus(args.manifest, args.out, tokenizer=SimpleTokenizer()) print(json.dumps(metadata, indent=2)) return 0 if __name__ == "__main__": raise SystemExit(main())