#!/usr/bin/env python3 # -*- coding: utf-8 -*- """ test_dynamic_transmutation.py Direct verification of the Sovereign Dynamic Farāhīdian Transmutation Engine. Tests arbitrary seen & unseen classical Arabic propositions without any hardcoded dictionary. """ import sys import json import time from pathlib import Path from typing import Dict, List, Any, Tuple, Optional import torch import torch.nn.functional as F from safetensors.torch import load_file BASE_DIR = Path('/workspace/rootformer_v12') V18_DIR = BASE_DIR / 'v18_next_root_morph' DATA_DIR = V18_DIR / 'data' CKPT_DIR = V18_DIR / 'checkpoints' sys.path.insert(0, str(BASE_DIR)) sys.path.insert(0, str(BASE_DIR / 'v17_deepseek_flash')) sys.path.insert(0, str(V18_DIR)) from models.unified_rootformer_v12 import UnifiedRootformerV12 from deepseek_v4_1_flash_model import UnifiedRootformerV17_DeepSeekFlash from neural_transmuter_head import NeuralFarahidianTransmuterHead from models.morphemic_tokenizer_v12_arabic import PureArabicMorphemicTokenizerV12 from farahidian_dynamic_compiler import FarahidianDynamicCompiler class DynamicBasranGuidedTransmuter: """ Sovereign Decoding Engine implementing Sībawayh's Dynamic Hard Exclusion Mask over Layer 14 Arabic Representations using Farāhīdian Morpho-Syntactic Compilation. """ def __init__( self, transmuter_head: Any, compiler: FarahidianDynamicCompiler, token_to_id: Dict[str, int], id_to_token: Dict[int, str] ): self.head = transmuter_head self.compiler = compiler self.token_to_id = token_to_id self.id_to_token = id_to_token self.device = next(transmuter_head.parameters()).device self.bos_id = token_to_id.get('', 2) self.eos_id = token_to_id.get('', 3) @torch.no_grad() def transmute( self, hidden_states: torch.Tensor, ar_text: str, temperature: float = 0.0 ) -> Tuple[List[str], str]: """ Executes dynamic transmutation using Sībawayh's Hard Exclusion Mask over the dynamically compiled Naẓm slots. """ self.head.eval() memory_l14 = self.head.extract_layer14_states(hidden_states).to(self.head.concept_embeddings.weight.dtype) # 1. Compile Farāhīdian Naẓm Slots dynamically slots = self.compiler.compile_dynamic_naẓm_plan(ar_text) if not slots: # Fallback if unparsable return ["the"], "FALLBACK" curr_ids = torch.tensor([[self.bos_id]], dtype=torch.long, device=self.device) generated_tokens: List[str] = [] # Iterate through the dynamically compiled slots for step, slot_candidates in enumerate(slots): seq_len = curr_ids.shape[1] x = self.head.concept_embeddings(curr_ids) causal_mask = torch.triu( torch.full((seq_len, seq_len), float('-inf'), device=self.device, dtype=x.dtype), diagonal=1 ) for layer in self.head.decoder_layers: x = layer(x, memory_l14, causal_mask=causal_mask) x = self.head.final_norm(x) next_logits = F.linear( x[:, -1, :], self.head.concept_embeddings.weight, self.head.output_bias )[0] # 2. Sībawayh Hard Grammatical Exclusion Mask for this Slot allowed_ids = self.compiler.get_valid_token_ids_for_slot(slot_candidates) if not allowed_ids: # If slot has no direct match, continue continue slot_mask = torch.full_like(next_logits, float('-inf')) for aid in allowed_ids: slot_mask[aid] = 0.0 masked_logits = next_logits + slot_mask if temperature > 0.0: probs = F.softmax(masked_logits / temperature, dim=-1) next_id = torch.multinomial(probs, num_samples=1).item() else: next_id = torch.argmax(masked_logits, dim=-1).item() token_str = self.id_to_token.get(next_id, '') if token_str and not token_str.startswith('<'): generated_tokens.append(token_str) curr_ids = torch.cat([curr_ids, torch.tensor([[next_id]], device=self.device)], dim=1) # Sībawayh Inqiṭāʿ al-ʿAmal: Constituents saturated -> Terminate return generated_tokens, "DYNAMIC_BASRAN_NAZM" def run_test(): device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') print("=" * 75) print(" VERIFYING SOVEREIGN DYNAMIC FARĀHĪDIAN TRANSMUTATION ENGINE") print(" Zero Lookup Dictionaries | Pure Morpho-Syntactic Compilation") print("=" * 75) vocab_file = DATA_DIR / 'concept_vocabulary.json' with open(vocab_file) as f: vdata = json.load(f) token_to_id = vdata['token_to_id'] id_to_token = {int(k): v for k, v in vdata['id_to_token'].items()} with open(DATA_DIR / 'root_concept_prior.json') as f: root_prior = json.load(f) blueprint_path = BASE_DIR / 'data/rootformer_v12_arabic_blueprint.json' tok = PureArabicMorphemicTokenizerV12(blueprint_path=blueprint_path) compiler = FarahidianDynamicCompiler(token_to_id, id_to_token, tok, root_prior) # Load 24-Layer Backbone base_model = UnifiedRootformerV12( blueprint_path=str(blueprint_path), base_model_name='Qwen/Qwen2.5-0.5B', device=device, dtype=torch.bfloat16 ).to(device) flash_model = UnifiedRootformerV17_DeepSeekFlash( base_v12_model=base_model, device=device, dtype=torch.bfloat16 ).to(device) master_ckpt = CKPT_DIR / 'rootformer_v18_grand_bilingual_master.safetensors' flash_model.load_state_dict(load_file(str(master_ckpt)), strict=True) flash_model.eval() # Load Transmuter Heads (Test both v18.1 and v18.3) d_model = getattr(base_model.backbone.config, 'hidden_size', 896) # Checkpoint v18.1 t_v18_1 = NeuralFarahidianTransmuterHead( d_model=d_model, d_concept=512, concept_vocab_size=vdata['vocab_size'], num_roots=9114, num_decoder_layers=2, dropout=0.0 ).to(device, torch.bfloat16) t_v18_1.load_state_dict(load_file(str(CKPT_DIR / 'rootformer_v18_neural_transmuter_master.safetensors'))) t_v18_1.eval() # Checkpoint v18.3 t_v18_3 = NeuralFarahidianTransmuterHead( d_model=d_model, d_concept=512, concept_vocab_size=vdata['vocab_size'], num_roots=9114, num_decoder_layers=2, dropout=0.0 ).to(device, torch.bfloat16) t_v18_3.load_state_dict(load_file(str(CKPT_DIR / 'rootformer_v18_3_neural_transmuter_master.safetensors'))) t_v18_3.eval() engine_v18_1 = DynamicBasranGuidedTransmuter(t_v18_1, compiler, token_to_id, id_to_token) engine_v18_3 = DynamicBasranGuidedTransmuter(t_v18_3, compiler, token_to_id, id_to_token) # 10 Diverse Classical Propositions (Seen & Unseen) test_suite = [ ("المشقة تجلب التيسير", "Hardship brings about facilitation."), ("اليقين لا يزول بالشك", "Certainty is not eliminated by doubt."), ("إنما الأعمال بالنيات", "Actions are only by intentions."), ("العدم لا يكون علة للوجود", "Nonexistence cannot be a cause for existence."), ("الوجود ينقسم إلى واجب وممكن وممتنع", "Existence is divided into necessary, contingent, and impossible."), ("العلم نور", "Knowledge is light."), ("الصبر نصف الإيمان", "Patience is half of faith."), ("كل حادث مسبوق بمادة ومدة", "Every originated entity is preceded by matter and duration."), ("من عرف نفسه فقد عرف ربه", "Whosoever knows their soul knows their Lord."), ("العمران البشري ضروري للجنس الإنساني", "Human civilization is necessary for the human species.") ] print("\nRunning Dynamic Transmutation across test suite...\n") for idx, (ar, ref) in enumerate(test_suite, 1): enc = base_model.tokenizer.encode(ar) t_in = torch.tensor([enc], dtype=torch.long, device=device) with torch.no_grad(): b_out = base_model.backbone(input_ids=t_in, output_hidden_states=True, return_dict=True) h14 = b_out.hidden_states[14] out_18_1, mode1 = engine_v18_1.transmute(h14, ar) out_18_3, mode3 = engine_v18_3.transmute(h14, ar) print(f"[{idx:2d}] ARABIC : {ar}") print(f" REF : {ref}") print(f" v18.1 : {' '.join(out_18_1)}") print(f" v18.3 : {' '.join(out_18_3)}") print("-" * 75) if __name__ == '__main__': run_test()