on_the_way / tools /export_rivinity_tokenizer.py
RANITBAG's picture
2nd run
1a1fc1f verified
Raw History Blame Contribute Delete
2.75 kB
#!/usr/bin/env python3
"""export_rivinity_tokenizer.py -- assemble the production tokenizer export dir.
Takes a trained tokenizer dir (tokenizer.json) and produces a complete export:
tokenizer.json (control + reasoning tokens registered)
tokenizer_config.json (class marker, model_max_length, chat template)
special_tokens_map.json
rivinity_tokenizer_metadata.json (resolved ids, template version, context)
Usage (repo root):
python tools/export_rivinity_tokenizer.py --tokenizer-dir tokenizer_out/T48 \
--out tokenizer_out/Rivinity-T48
"""
from __future__ import annotations
import argparse
import json
import shutil
import sys
from pathlib import Path
REPO_ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(REPO_ROOT))
from rivinity.tokenizer import (DEFAULT_CHAT_TEMPLATE, TEMPLATE_VERSION,
ensure_control_tokens, load_tokenizer,
tokenizer_metadata)
def main() -> int:
parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
parser.add_argument("--tokenizer-dir", type=Path, required=True)
parser.add_argument("--out", type=Path, required=True)
parser.add_argument("--model-max-length", type=int, default=32768)
args = parser.parse_args()
tok = load_tokenizer(args.tokenizer_dir)
ids = ensure_control_tokens(tok)
meta = tokenizer_metadata(tok)
args.out.mkdir(parents=True, exist_ok=True)
tok.save(str(args.out / "tokenizer.json"))
(args.out / "tokenizer_config.json").write_text(json.dumps({
"tokenizer_class": "RivinityTokenizer",
"model_max_length": args.model_max_length,
"template_version": TEMPLATE_VERSION,
"chat_template": DEFAULT_CHAT_TEMPLATE,
"added_tokens_decoder": {str(i): {"content": t, "special": True}
for t, i in sorted(ids.items(), key=lambda kv: kv[1])},
}, indent=2, ensure_ascii=False) + "\n", encoding="utf-8")
(args.out / "special_tokens_map.json").write_text(json.dumps({
"unk_token": "<unk>", "bos_token": "<s>", "eos_token": "</s>",
"additional_special_tokens": sorted(ids),
}, indent=2, ensure_ascii=False) + "\n", encoding="utf-8")
meta["context"]["model_max_length"] = args.model_max_length
(args.out / "rivinity_tokenizer_metadata.json").write_text(
json.dumps(meta, indent=2, ensure_ascii=False) + "\n", encoding="utf-8")
print(f"export written -> {args.out} (vocab {meta['vocab_size']}, "
f"template {TEMPLATE_VERSION}, model_max_length {args.model_max_length})")
return 0
if __name__ == "__main__":
raise SystemExit(main())