#!/usr/bin/env python3 """Run upstream conversion with a lossless Transformers 5 -> 4 config adapter. The pinned upstream requirements use Transformers 4.57.6. Google's tokenizer configuration uses Transformers 5's list-valued extra_special_tokens and model_specific_special_tokens mapping. Supply the equivalent Transformers 4 arguments in memory; leave all checkpoint and upstream tool files untouched. """ import json import pathlib import runpy import sys from tokenizers import Tokenizer from transformers import AutoTokenizer root = pathlib.Path(__file__).resolve().parents[1] original_loader = AutoTokenizer.from_pretrained def load_compatible(model_path, *args, **kwargs): path = pathlib.Path(model_path) config = json.loads((path / "tokenizer_config.json").read_text()) extra = config.get("extra_special_tokens") if isinstance(extra, list): kwargs["extra_special_tokens"] = config["model_specific_special_tokens"] kwargs["additional_special_tokens"] = extra tokenizer = original_loader(model_path, *args, **kwargs) original = Tokenizer.from_file(str(path / "tokenizer.json")) assert tokenizer.get_vocab() == original.get_vocab(), "Tokenizer vocabulary or IDs changed" for name, token in config.get("model_specific_special_tokens", {}).items(): assert tokenizer.convert_tokens_to_ids(token) == original.token_to_id(token), name assert token in tokenizer.all_special_tokens, name print(f"Tokenizer compatibility audit: all {len(tokenizer)} token strings/IDs preserved; all model-specific special tokens preserved.", flush=True) return tokenizer AutoTokenizer.from_pretrained = staticmethod(load_compatible) sys.path.insert(0, str(root / "llama.cpp")) runpy.run_path(str(root / "llama.cpp/convert_hf_to_gguf.py"), run_name="__main__")