algo-v1-codepoint / scripts /build_corpus.py
Miruamel's picture
Publish algo v1 (code-point tokenizer): 38.8M params, 16.4M tokens, loss 2.10
05f5056 verified
Raw History Blame Contribute Delete
1.15 kB
#!/usr/bin/env python
"""Build the real training corpus into a memory-mapped token cache.
python scripts/build_corpus.py [--manifest data/manifest.json] [--out DIR]
Fetches every enabled source in the manifest, strips it to prose, deduplicates,
checks it against the held-out eval set, and writes one uint16 token file that
training memory-maps. The download is tens of megabytes; the resulting cache
is not committed.
"""
from __future__ import annotations
import argparse
import json
import sys
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
from algo.corpus import CACHE_DIR, build_corpus # noqa: E402
from algo.data import MANIFEST_PATH, SimpleTokenizer # noqa: E402
def main() -> int:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--manifest", default=str(MANIFEST_PATH))
parser.add_argument("--out", default=str(CACHE_DIR))
args = parser.parse_args()
metadata = build_corpus(args.manifest, args.out, tokenizer=SimpleTokenizer())
print(json.dumps(metadata, indent=2))
return 0
if __name__ == "__main__":
raise SystemExit(main())