Download scripts/build_corpus.py from Miruamel/algo-v1-codepoint: direct link, hf CLI and curl.
- Browser
- Download file 1.15 kB
-
https://huggingface.co/Miruamel/algo-v1-codepoint/resolve/main/scripts/build_corpus.py
- Command line
-
hf download hf://Miruamel/algo-v1-codepoint/scripts/build_corpus.py
-
curl -L -o build_corpus.py https://huggingface.co/Miruamel/algo-v1-codepoint/resolve/main/scripts/build_corpus.py
1.15 kB
| #!/usr/bin/env python | |
| """Build the real training corpus into a memory-mapped token cache. | |
| python scripts/build_corpus.py [--manifest data/manifest.json] [--out DIR] | |
| Fetches every enabled source in the manifest, strips it to prose, deduplicates, | |
| checks it against the held-out eval set, and writes one uint16 token file that | |
| training memory-maps. The download is tens of megabytes; the resulting cache | |
| is not committed. | |
| """ | |
| from __future__ import annotations | |
| import argparse | |
| import json | |
| import sys | |
| from pathlib import Path | |
| sys.path.insert(0, str(Path(__file__).resolve().parents[1])) | |
| from algo.corpus import CACHE_DIR, build_corpus # noqa: E402 | |
| from algo.data import MANIFEST_PATH, SimpleTokenizer # noqa: E402 | |
| def main() -> int: | |
| parser = argparse.ArgumentParser(description=__doc__) | |
| parser.add_argument("--manifest", default=str(MANIFEST_PATH)) | |
| parser.add_argument("--out", default=str(CACHE_DIR)) | |
| args = parser.parse_args() | |
| metadata = build_corpus(args.manifest, args.out, tokenizer=SimpleTokenizer()) | |
| print(json.dumps(metadata, indent=2)) | |
| return 0 | |
| if __name__ == "__main__": | |
| raise SystemExit(main()) | |