#!/usr/bin/env python3 """export_rivinity_tokenizer.py -- assemble the production tokenizer export dir. Takes a trained tokenizer dir (tokenizer.json) and produces a complete export: tokenizer.json (control + reasoning tokens registered) tokenizer_config.json (class marker, model_max_length, chat template) special_tokens_map.json rivinity_tokenizer_metadata.json (resolved ids, template version, context) Usage (repo root): python tools/export_rivinity_tokenizer.py --tokenizer-dir tokenizer_out/T48 \ --out tokenizer_out/Rivinity-T48 """ from __future__ import annotations import argparse import json import shutil import sys from pathlib import Path REPO_ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(REPO_ROOT)) from rivinity.tokenizer import (DEFAULT_CHAT_TEMPLATE, TEMPLATE_VERSION, ensure_control_tokens, load_tokenizer, tokenizer_metadata) def main() -> int: parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) parser.add_argument("--tokenizer-dir", type=Path, required=True) parser.add_argument("--out", type=Path, required=True) parser.add_argument("--model-max-length", type=int, default=32768) args = parser.parse_args() tok = load_tokenizer(args.tokenizer_dir) ids = ensure_control_tokens(tok) meta = tokenizer_metadata(tok) args.out.mkdir(parents=True, exist_ok=True) tok.save(str(args.out / "tokenizer.json")) (args.out / "tokenizer_config.json").write_text(json.dumps({ "tokenizer_class": "RivinityTokenizer", "model_max_length": args.model_max_length, "template_version": TEMPLATE_VERSION, "chat_template": DEFAULT_CHAT_TEMPLATE, "added_tokens_decoder": {str(i): {"content": t, "special": True} for t, i in sorted(ids.items(), key=lambda kv: kv[1])}, }, indent=2, ensure_ascii=False) + "\n", encoding="utf-8") (args.out / "special_tokens_map.json").write_text(json.dumps({ "unk_token": "", "bos_token": "", "eos_token": "", "additional_special_tokens": sorted(ids), }, indent=2, ensure_ascii=False) + "\n", encoding="utf-8") meta["context"]["model_max_length"] = args.model_max_length (args.out / "rivinity_tokenizer_metadata.json").write_text( json.dumps(meta, indent=2, ensure_ascii=False) + "\n", encoding="utf-8") print(f"export written -> {args.out} (vocab {meta['vocab_size']}, " f"template {TEMPLATE_VERSION}, model_max_length {args.model_max_length})") return 0 if __name__ == "__main__": raise SystemExit(main())