rootformer-v18-basran-transmute / scripts /test_dynamic_transmutation.py
enver's picture
Clean up repository layout, add official model card, config.json, quickstart, and directory indices
2f0bafa verified
Raw History Blame Contribute Delete
8.84 kB
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
test_dynamic_transmutation.py
Direct verification of the Sovereign Dynamic Farāhīdian Transmutation Engine.
Tests arbitrary seen & unseen classical Arabic propositions without any hardcoded dictionary.
"""
import sys
import json
import time
from pathlib import Path
from typing import Dict, List, Any, Tuple, Optional
import torch
import torch.nn.functional as F
from safetensors.torch import load_file
BASE_DIR = Path('/workspace/rootformer_v12')
V18_DIR = BASE_DIR / 'v18_next_root_morph'
DATA_DIR = V18_DIR / 'data'
CKPT_DIR = V18_DIR / 'checkpoints'
sys.path.insert(0, str(BASE_DIR))
sys.path.insert(0, str(BASE_DIR / 'v17_deepseek_flash'))
sys.path.insert(0, str(V18_DIR))
from models.unified_rootformer_v12 import UnifiedRootformerV12
from deepseek_v4_1_flash_model import UnifiedRootformerV17_DeepSeekFlash
from neural_transmuter_head import NeuralFarahidianTransmuterHead
from models.morphemic_tokenizer_v12_arabic import PureArabicMorphemicTokenizerV12
from farahidian_dynamic_compiler import FarahidianDynamicCompiler
class DynamicBasranGuidedTransmuter:
"""
Sovereign Decoding Engine implementing Sībawayh's Dynamic Hard Exclusion Mask
over Layer 14 Arabic Representations using Farāhīdian Morpho-Syntactic Compilation.
"""
def __init__(
self,
transmuter_head: Any,
compiler: FarahidianDynamicCompiler,
token_to_id: Dict[str, int],
id_to_token: Dict[int, str]
):
self.head = transmuter_head
self.compiler = compiler
self.token_to_id = token_to_id
self.id_to_token = id_to_token
self.device = next(transmuter_head.parameters()).device
self.bos_id = token_to_id.get('<BOS>', 2)
self.eos_id = token_to_id.get('<EOS>', 3)
@torch.no_grad()
def transmute(
self,
hidden_states: torch.Tensor,
ar_text: str,
temperature: float = 0.0
) -> Tuple[List[str], str]:
"""
Executes dynamic transmutation using Sībawayh's Hard Exclusion Mask
over the dynamically compiled Naẓm slots.
"""
self.head.eval()
memory_l14 = self.head.extract_layer14_states(hidden_states).to(self.head.concept_embeddings.weight.dtype)
# 1. Compile Farāhīdian Naẓm Slots dynamically
slots = self.compiler.compile_dynamic_naẓm_plan(ar_text)
if not slots:
# Fallback if unparsable
return ["the"], "FALLBACK"
curr_ids = torch.tensor([[self.bos_id]], dtype=torch.long, device=self.device)
generated_tokens: List[str] = []
# Iterate through the dynamically compiled slots
for step, slot_candidates in enumerate(slots):
seq_len = curr_ids.shape[1]
x = self.head.concept_embeddings(curr_ids)
causal_mask = torch.triu(
torch.full((seq_len, seq_len), float('-inf'), device=self.device, dtype=x.dtype),
diagonal=1
)
for layer in self.head.decoder_layers:
x = layer(x, memory_l14, causal_mask=causal_mask)
x = self.head.final_norm(x)
next_logits = F.linear(
x[:, -1, :],
self.head.concept_embeddings.weight,
self.head.output_bias
)[0]
# 2. Sībawayh Hard Grammatical Exclusion Mask for this Slot
allowed_ids = self.compiler.get_valid_token_ids_for_slot(slot_candidates)
if not allowed_ids:
# If slot has no direct match, continue
continue
slot_mask = torch.full_like(next_logits, float('-inf'))
for aid in allowed_ids:
slot_mask[aid] = 0.0
masked_logits = next_logits + slot_mask
if temperature > 0.0:
probs = F.softmax(masked_logits / temperature, dim=-1)
next_id = torch.multinomial(probs, num_samples=1).item()
else:
next_id = torch.argmax(masked_logits, dim=-1).item()
token_str = self.id_to_token.get(next_id, '')
if token_str and not token_str.startswith('<'):
generated_tokens.append(token_str)
curr_ids = torch.cat([curr_ids, torch.tensor([[next_id]], device=self.device)], dim=1)
# Sībawayh Inqiṭāʿ al-ʿAmal: Constituents saturated -> Terminate
return generated_tokens, "DYNAMIC_BASRAN_NAZM"
def run_test():
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
print("=" * 75)
print(" VERIFYING SOVEREIGN DYNAMIC FARĀHĪDIAN TRANSMUTATION ENGINE")
print(" Zero Lookup Dictionaries | Pure Morpho-Syntactic Compilation")
print("=" * 75)
vocab_file = DATA_DIR / 'concept_vocabulary.json'
with open(vocab_file) as f:
vdata = json.load(f)
token_to_id = vdata['token_to_id']
id_to_token = {int(k): v for k, v in vdata['id_to_token'].items()}
with open(DATA_DIR / 'root_concept_prior.json') as f:
root_prior = json.load(f)
blueprint_path = BASE_DIR / 'data/rootformer_v12_arabic_blueprint.json'
tok = PureArabicMorphemicTokenizerV12(blueprint_path=blueprint_path)
compiler = FarahidianDynamicCompiler(token_to_id, id_to_token, tok, root_prior)
# Load 24-Layer Backbone
base_model = UnifiedRootformerV12(
blueprint_path=str(blueprint_path),
base_model_name='Qwen/Qwen2.5-0.5B',
device=device,
dtype=torch.bfloat16
).to(device)
flash_model = UnifiedRootformerV17_DeepSeekFlash(
base_v12_model=base_model,
device=device,
dtype=torch.bfloat16
).to(device)
master_ckpt = CKPT_DIR / 'rootformer_v18_grand_bilingual_master.safetensors'
flash_model.load_state_dict(load_file(str(master_ckpt)), strict=True)
flash_model.eval()
# Load Transmuter Heads (Test both v18.1 and v18.3)
d_model = getattr(base_model.backbone.config, 'hidden_size', 896)
# Checkpoint v18.1
t_v18_1 = NeuralFarahidianTransmuterHead(
d_model=d_model, d_concept=512, concept_vocab_size=vdata['vocab_size'],
num_roots=9114, num_decoder_layers=2, dropout=0.0
).to(device, torch.bfloat16)
t_v18_1.load_state_dict(load_file(str(CKPT_DIR / 'rootformer_v18_neural_transmuter_master.safetensors')))
t_v18_1.eval()
# Checkpoint v18.3
t_v18_3 = NeuralFarahidianTransmuterHead(
d_model=d_model, d_concept=512, concept_vocab_size=vdata['vocab_size'],
num_roots=9114, num_decoder_layers=2, dropout=0.0
).to(device, torch.bfloat16)
t_v18_3.load_state_dict(load_file(str(CKPT_DIR / 'rootformer_v18_3_neural_transmuter_master.safetensors')))
t_v18_3.eval()
engine_v18_1 = DynamicBasranGuidedTransmuter(t_v18_1, compiler, token_to_id, id_to_token)
engine_v18_3 = DynamicBasranGuidedTransmuter(t_v18_3, compiler, token_to_id, id_to_token)
# 10 Diverse Classical Propositions (Seen & Unseen)
test_suite = [
("المشقة تجلب التيسير", "Hardship brings about facilitation."),
("اليقين لا يزول بالشك", "Certainty is not eliminated by doubt."),
("إنما الأعمال بالنيات", "Actions are only by intentions."),
("العدم لا يكون علة للوجود", "Nonexistence cannot be a cause for existence."),
("الوجود ينقسم إلى واجب وممكن وممتنع", "Existence is divided into necessary, contingent, and impossible."),
("العلم نور", "Knowledge is light."),
("الصبر نصف الإيمان", "Patience is half of faith."),
("كل حادث مسبوق بمادة ومدة", "Every originated entity is preceded by matter and duration."),
("من عرف نفسه فقد عرف ربه", "Whosoever knows their soul knows their Lord."),
("العمران البشري ضروري للجنس الإنساني", "Human civilization is necessary for the human species.")
]
print("\nRunning Dynamic Transmutation across test suite...\n")
for idx, (ar, ref) in enumerate(test_suite, 1):
enc = base_model.tokenizer.encode(ar)
t_in = torch.tensor([enc], dtype=torch.long, device=device)
with torch.no_grad():
b_out = base_model.backbone(input_ids=t_in, output_hidden_states=True, return_dict=True)
h14 = b_out.hidden_states[14]
out_18_1, mode1 = engine_v18_1.transmute(h14, ar)
out_18_3, mode3 = engine_v18_3.transmute(h14, ar)
print(f"[{idx:2d}] ARABIC : {ar}")
print(f" REF : {ref}")
print(f" v18.1 : {' '.join(out_18_1)}")
print(f" v18.3 : {' '.join(out_18_3)}")
print("-" * 75)
if __name__ == '__main__':
run_test()