Upload testing/convert_gpt2.py with huggingface_hub

testing/convert_gpt2.py

@@ -0,0 +1,33 @@
+"""Correct HF->GGUF f16 conversion for the rebuilt v4 ByteLevel tokenizer:
+force pre='gpt-2' (verified full token parity) instead of the wrong 'sarvam-moe'.
+Keeps the BYTE-retype patch for <0xXX> byte-fallback tokens."""
+import os, sys, re, gguf
+LL = "/workspace/llm/llama.cpp"
+SRC = "/workspace/llm/kos-v4-conv"
+OUT = "/workspace/llm/kos-v4-gguf/kosv4-f16.gguf"
+sys.path.insert(0, LL); os.chdir(LL)
+from conversion.base import TextModel
+
+def _pre(self, tokenizer):
+    return "gpt-2"
+TextModel.get_vocab_base_pre = _pre
+
+_byte = re.compile(r"^<0x[0-9A-Fa-f]{2}>$")
+_orig_base = TextModel.get_vocab_base
+def _base_patched(self):
+    tokens, toktypes, tokpre = _orig_base(self)
+    n = 0
+    for i, t in enumerate(tokens):
+        if _byte.match(t) and toktypes[i] != gguf.TokenType.BYTE:
+            toktypes[i] = gguf.TokenType.BYTE; n += 1
+    if n: print(f"[patch] retyped {n} byte-fallback tokens as BYTE", file=sys.stderr)
+    return tokens, toktypes, tokpre
+TextModel.get_vocab_base = _base_patched
+
+import convert_hf_to_gguf as conv
+sys.argv = ["convert_hf_to_gguf.py", "--outfile", OUT, "--outtype", "f16", SRC]
+try:
+    conv.main()
+except SystemExit as e:
+    if e.code not in (None, 0): raise
+print("[done] wrote", OUT)