enver's picture
Add official Rootformer Tokenizer & AutoClass integration (quickstart.py)
8b4a7ab verified
Raw History Blame Contribute Delete
2.9 kB
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Rootformer v18.8 Quickstart Demo
Official inference pipeline using Hugging Face AutoClasses:
- AutoTokenizer: RootformerTokenizer (Farāhīdian radical decomposition & Basran syntax)
- AutoModelForCausalLM: RootformerForCausalLM (24-layer transformer + NRMP factorized head)
"""
import torch
from transformers import AutoTokenizer, AutoModelForCausalLM
MODEL_ID = "enver/rootformer-v18-basran-transmute"
def main():
device = "cuda" if torch.cuda.is_available() else "cpu"
print(f"[*] Initializing Rootformer v18.8 on {device}...")
# 1. Load Farāhīdian Morphemic Tokenizer
tokenizer = AutoTokenizer.from_pretrained(MODEL_ID, trust_remote_code=True)
print(f"[+] Loaded Tokenizer: {tokenizer.__class__.__name__} ({tokenizer.vocab_size} radical roots)")
# 2. Load Deep Unfrozen Foundation Model
model = AutoModelForCausalLM.from_pretrained(
MODEL_ID,
trust_remote_code=True,
torch_dtype=torch.bfloat16 if torch.cuda.is_available() else torch.float32
).to(device)
model.eval()
print(f"[+] Loaded Model: {model.__class__.__name__} (~384M parameters, 135M unfrozen)")
# 3. Test Prompt
prompt = "العلم نور والجهل ظلام"
print(f"\n[Prompt Input]: «{prompt}»")
inputs = tokenizer(prompt, return_tensors="pt").to(device)
print(f"[Morphemic Decomposition]:")
tuples = tokenizer.encode_morphemes(prompt)
for word, tup in zip(prompt.split(), tuples):
p, r, wz, s = tup
p_str = tokenizer.id2prefix.get(p, '')
r_str = tokenizer.id2root.get(r, '')
w_str = tokenizer.id2wazn.get(wz, '')
s_str = tokenizer.id2suffix.get(s, '')
print(f" Word: {word:<12} -> Root: {r_str:<8} Wazn: {w_str:<12} Prefix: {p_str:<4} Suffix: {s_str:<4}")
# 4. Predict Next Word & Generate Continuation
with torch.no_grad():
outputs = model(**inputs)
next_r = torch.argmax(outputs.root_logits[:, -1, :], dim=-1).item()
next_w = torch.argmax(outputs.wazn_logits[:, -1, :], dim=-1).item()
next_p = torch.argmax(outputs.prefix_logits[:, -1, :], dim=-1).item()
next_s = torch.argmax(outputs.suffix_logits[:, -1, :], dim=-1).item()
next_word = tokenizer.decode_word(next_p, next_r, next_w, next_s)
print(f"\n[Next-Word Prediction]: «{next_word}» (Root: {tokenizer.id2root.get(next_r)})")
print("[Generating Continuation]...")
gen_tuples = model.generate_morphemes(
p_ids=inputs['p_ids'],
r_ids=inputs['r_ids'],
w_ids=inputs['w_ids'],
s_ids=inputs['s_ids'],
max_new_tokens=8,
temperature=0.7
)
continuation = tokenizer.decode(gen_tuples)
print(f"[Full Realized Continuation]: «{prompt} {continuation}»")
if __name__ == "__main__":
main()