Download tools/export_rivinity_tokenizer.py from RANITBAG/on_the_way: direct link, hf CLI and curl.
- Browser
- Download file 2.75 kB
-
https://huggingface.co/RANITBAG/on_the_way/resolve/main/tools/export_rivinity_tokenizer.py
- Command line
-
hf download hf://RANITBAG/on_the_way/tools/export_rivinity_tokenizer.py
-
curl -L -o export_rivinity_tokenizer.py https://huggingface.co/RANITBAG/on_the_way/resolve/main/tools/export_rivinity_tokenizer.py
2.75 kB
| #!/usr/bin/env python3 | |
| """export_rivinity_tokenizer.py -- assemble the production tokenizer export dir. | |
| Takes a trained tokenizer dir (tokenizer.json) and produces a complete export: | |
| tokenizer.json (control + reasoning tokens registered) | |
| tokenizer_config.json (class marker, model_max_length, chat template) | |
| special_tokens_map.json | |
| rivinity_tokenizer_metadata.json (resolved ids, template version, context) | |
| Usage (repo root): | |
| python tools/export_rivinity_tokenizer.py --tokenizer-dir tokenizer_out/T48 \ | |
| --out tokenizer_out/Rivinity-T48 | |
| """ | |
| from __future__ import annotations | |
| import argparse | |
| import json | |
| import shutil | |
| import sys | |
| from pathlib import Path | |
| REPO_ROOT = Path(__file__).resolve().parents[1] | |
| sys.path.insert(0, str(REPO_ROOT)) | |
| from rivinity.tokenizer import (DEFAULT_CHAT_TEMPLATE, TEMPLATE_VERSION, | |
| ensure_control_tokens, load_tokenizer, | |
| tokenizer_metadata) | |
| def main() -> int: | |
| parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) | |
| parser.add_argument("--tokenizer-dir", type=Path, required=True) | |
| parser.add_argument("--out", type=Path, required=True) | |
| parser.add_argument("--model-max-length", type=int, default=32768) | |
| args = parser.parse_args() | |
| tok = load_tokenizer(args.tokenizer_dir) | |
| ids = ensure_control_tokens(tok) | |
| meta = tokenizer_metadata(tok) | |
| args.out.mkdir(parents=True, exist_ok=True) | |
| tok.save(str(args.out / "tokenizer.json")) | |
| (args.out / "tokenizer_config.json").write_text(json.dumps({ | |
| "tokenizer_class": "RivinityTokenizer", | |
| "model_max_length": args.model_max_length, | |
| "template_version": TEMPLATE_VERSION, | |
| "chat_template": DEFAULT_CHAT_TEMPLATE, | |
| "added_tokens_decoder": {str(i): {"content": t, "special": True} | |
| for t, i in sorted(ids.items(), key=lambda kv: kv[1])}, | |
| }, indent=2, ensure_ascii=False) + "\n", encoding="utf-8") | |
| (args.out / "special_tokens_map.json").write_text(json.dumps({ | |
| "unk_token": "<unk>", "bos_token": "<s>", "eos_token": "</s>", | |
| "additional_special_tokens": sorted(ids), | |
| }, indent=2, ensure_ascii=False) + "\n", encoding="utf-8") | |
| meta["context"]["model_max_length"] = args.model_max_length | |
| (args.out / "rivinity_tokenizer_metadata.json").write_text( | |
| json.dumps(meta, indent=2, ensure_ascii=False) + "\n", encoding="utf-8") | |
| print(f"export written -> {args.out} (vocab {meta['vocab_size']}, " | |
| f"template {TEMPLATE_VERSION}, model_max_length {args.model_max_length})") | |
| return 0 | |
| if __name__ == "__main__": | |
| raise SystemExit(main()) |