File size: 8,841 Bytes
d517cde
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
test_dynamic_transmutation.py
Direct verification of the Sovereign Dynamic Farāhīdian Transmutation Engine.
Tests arbitrary seen & unseen classical Arabic propositions without any hardcoded dictionary.
"""

import sys
import json
import time
from pathlib import Path
from typing import Dict, List, Any, Tuple, Optional

import torch
import torch.nn.functional as F
from safetensors.torch import load_file

BASE_DIR = Path('/workspace/rootformer_v12')
V18_DIR = BASE_DIR / 'v18_next_root_morph'
DATA_DIR = V18_DIR / 'data'
CKPT_DIR = V18_DIR / 'checkpoints'

sys.path.insert(0, str(BASE_DIR))
sys.path.insert(0, str(BASE_DIR / 'v17_deepseek_flash'))
sys.path.insert(0, str(V18_DIR))

from models.unified_rootformer_v12 import UnifiedRootformerV12
from deepseek_v4_1_flash_model import UnifiedRootformerV17_DeepSeekFlash
from neural_transmuter_head import NeuralFarahidianTransmuterHead
from models.morphemic_tokenizer_v12_arabic import PureArabicMorphemicTokenizerV12
from farahidian_dynamic_compiler import FarahidianDynamicCompiler

class DynamicBasranGuidedTransmuter:
    """
    Sovereign Decoding Engine implementing Sībawayh's Dynamic Hard Exclusion Mask
    over Layer 14 Arabic Representations using Farāhīdian Morpho-Syntactic Compilation.
    """

    def __init__(
        self,
        transmuter_head: Any,
        compiler: FarahidianDynamicCompiler,
        token_to_id: Dict[str, int],
        id_to_token: Dict[int, str]
    ):
        self.head = transmuter_head
        self.compiler = compiler
        self.token_to_id = token_to_id
        self.id_to_token = id_to_token
        self.device = next(transmuter_head.parameters()).device

        self.bos_id = token_to_id.get('<BOS>', 2)
        self.eos_id = token_to_id.get('<EOS>', 3)

    @torch.no_grad()
    def transmute(
        self,
        hidden_states: torch.Tensor,
        ar_text: str,
        temperature: float = 0.0
    ) -> Tuple[List[str], str]:
        """
        Executes dynamic transmutation using Sībawayh's Hard Exclusion Mask
        over the dynamically compiled Naẓm slots.
        """
        self.head.eval()
        memory_l14 = self.head.extract_layer14_states(hidden_states).to(self.head.concept_embeddings.weight.dtype)

        # 1. Compile Farāhīdian Naẓm Slots dynamically
        slots = self.compiler.compile_dynamic_naẓm_plan(ar_text)
        if not slots:
            # Fallback if unparsable
            return ["the"], "FALLBACK"

        curr_ids = torch.tensor([[self.bos_id]], dtype=torch.long, device=self.device)
        generated_tokens: List[str] = []

        # Iterate through the dynamically compiled slots
        for step, slot_candidates in enumerate(slots):
            seq_len = curr_ids.shape[1]
            x = self.head.concept_embeddings(curr_ids)
            causal_mask = torch.triu(
                torch.full((seq_len, seq_len), float('-inf'), device=self.device, dtype=x.dtype),
                diagonal=1
            )

            for layer in self.head.decoder_layers:
                x = layer(x, memory_l14, causal_mask=causal_mask)
            x = self.head.final_norm(x)

            next_logits = F.linear(
                x[:, -1, :],
                self.head.concept_embeddings.weight,
                self.head.output_bias
            )[0]

            # 2. Sībawayh Hard Grammatical Exclusion Mask for this Slot
            allowed_ids = self.compiler.get_valid_token_ids_for_slot(slot_candidates)
            if not allowed_ids:
                # If slot has no direct match, continue
                continue

            slot_mask = torch.full_like(next_logits, float('-inf'))
            for aid in allowed_ids:
                slot_mask[aid] = 0.0

            masked_logits = next_logits + slot_mask

            if temperature > 0.0:
                probs = F.softmax(masked_logits / temperature, dim=-1)
                next_id = torch.multinomial(probs, num_samples=1).item()
            else:
                next_id = torch.argmax(masked_logits, dim=-1).item()

            token_str = self.id_to_token.get(next_id, '')
            if token_str and not token_str.startswith('<'):
                generated_tokens.append(token_str)

            curr_ids = torch.cat([curr_ids, torch.tensor([[next_id]], device=self.device)], dim=1)

        # Sībawayh Inqiṭāʿ al-ʿAmal: Constituents saturated -> Terminate
        return generated_tokens, "DYNAMIC_BASRAN_NAZM"

def run_test():
    device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
    print("=" * 75)
    print("  VERIFYING SOVEREIGN DYNAMIC FARĀHĪDIAN TRANSMUTATION ENGINE")
    print("  Zero Lookup Dictionaries | Pure Morpho-Syntactic Compilation")
    print("=" * 75)

    vocab_file = DATA_DIR / 'concept_vocabulary.json'
    with open(vocab_file) as f:
        vdata = json.load(f)
    token_to_id = vdata['token_to_id']
    id_to_token = {int(k): v for k, v in vdata['id_to_token'].items()}

    with open(DATA_DIR / 'root_concept_prior.json') as f:
        root_prior = json.load(f)

    blueprint_path = BASE_DIR / 'data/rootformer_v12_arabic_blueprint.json'
    tok = PureArabicMorphemicTokenizerV12(blueprint_path=blueprint_path)

    compiler = FarahidianDynamicCompiler(token_to_id, id_to_token, tok, root_prior)

    # Load 24-Layer Backbone
    base_model = UnifiedRootformerV12(
        blueprint_path=str(blueprint_path),
        base_model_name='Qwen/Qwen2.5-0.5B',
        device=device,
        dtype=torch.bfloat16
    ).to(device)

    flash_model = UnifiedRootformerV17_DeepSeekFlash(
        base_v12_model=base_model,
        device=device,
        dtype=torch.bfloat16
    ).to(device)

    master_ckpt = CKPT_DIR / 'rootformer_v18_grand_bilingual_master.safetensors'
    flash_model.load_state_dict(load_file(str(master_ckpt)), strict=True)
    flash_model.eval()

    # Load Transmuter Heads (Test both v18.1 and v18.3)
    d_model = getattr(base_model.backbone.config, 'hidden_size', 896)
    
    # Checkpoint v18.1
    t_v18_1 = NeuralFarahidianTransmuterHead(
        d_model=d_model, d_concept=512, concept_vocab_size=vdata['vocab_size'],
        num_roots=9114, num_decoder_layers=2, dropout=0.0
    ).to(device, torch.bfloat16)
    t_v18_1.load_state_dict(load_file(str(CKPT_DIR / 'rootformer_v18_neural_transmuter_master.safetensors')))
    t_v18_1.eval()

    # Checkpoint v18.3
    t_v18_3 = NeuralFarahidianTransmuterHead(
        d_model=d_model, d_concept=512, concept_vocab_size=vdata['vocab_size'],
        num_roots=9114, num_decoder_layers=2, dropout=0.0
    ).to(device, torch.bfloat16)
    t_v18_3.load_state_dict(load_file(str(CKPT_DIR / 'rootformer_v18_3_neural_transmuter_master.safetensors')))
    t_v18_3.eval()

    engine_v18_1 = DynamicBasranGuidedTransmuter(t_v18_1, compiler, token_to_id, id_to_token)
    engine_v18_3 = DynamicBasranGuidedTransmuter(t_v18_3, compiler, token_to_id, id_to_token)

    # 10 Diverse Classical Propositions (Seen & Unseen)
    test_suite = [
        ("المشقة تجلب التيسير", "Hardship brings about facilitation."),
        ("اليقين لا يزول بالشك", "Certainty is not eliminated by doubt."),
        ("إنما الأعمال بالنيات", "Actions are only by intentions."),
        ("العدم لا يكون علة للوجود", "Nonexistence cannot be a cause for existence."),
        ("الوجود ينقسم إلى واجب وممكن وممتنع", "Existence is divided into necessary, contingent, and impossible."),
        ("العلم نور", "Knowledge is light."),
        ("الصبر نصف الإيمان", "Patience is half of faith."),
        ("كل حادث مسبوق بمادة ومدة", "Every originated entity is preceded by matter and duration."),
        ("من عرف نفسه فقد عرف ربه", "Whosoever knows their soul knows their Lord."),
        ("العمران البشري ضروري للجنس الإنساني", "Human civilization is necessary for the human species.")
    ]

    print("\nRunning Dynamic Transmutation across test suite...\n")
    for idx, (ar, ref) in enumerate(test_suite, 1):
        enc = base_model.tokenizer.encode(ar)
        t_in = torch.tensor([enc], dtype=torch.long, device=device)
        with torch.no_grad():
            b_out = base_model.backbone(input_ids=t_in, output_hidden_states=True, return_dict=True)
            h14 = b_out.hidden_states[14]

            out_18_1, mode1 = engine_v18_1.transmute(h14, ar)
            out_18_3, mode3 = engine_v18_3.transmute(h14, ar)

        print(f"[{idx:2d}] ARABIC : {ar}")
        print(f"     REF    : {ref}")
        print(f"     v18.1  : {' '.join(out_18_1)}")
        print(f"     v18.3  : {' '.join(out_18_3)}")
        print("-" * 75)

if __name__ == '__main__':
    run_test()