Download ae.py from ChadlieOak/ae: direct link, hf CLI and curl.
- Browser
- Download file 20.8 kB
-
https://huggingface.co/ChadlieOak/ae/resolve/main/ae.py
- Command line
-
hf download hf://ChadlieOak/ae/ae.py
-
curl -L -o ae.py https://huggingface.co/ChadlieOak/ae/resolve/main/ae.py
20.8 kB
| import torch | |
| import torch.nn as nn | |
| import torch.nn.functional as F | |
| import torch.optim as optim | |
| from torch.utils.data import Dataset, DataLoader | |
| from torch.optim.lr_scheduler import CosineAnnealingWarmRestarts | |
| import math | |
| import random | |
| import numpy as np | |
| # Set random seed for reproducibility | |
| torch.manual_seed(42) | |
| np.random.seed(42) | |
| random.seed(42) | |
| # ============================================================================= | |
| # UPGRADE 1: Dynamic Fractal-Based Attention Mechanism | |
| # ============================================================================= | |
| class FractalAttentionalResonance(nn.Module): | |
| """ | |
| UPGRADED: The fractal resonance bias is now DYNAMICALLY generated based | |
| on the input sequence, making it content-aware. | |
| """ | |
| def __init__(self, dim, num_heads=4): | |
| super().__init__() | |
| self.dim = dim | |
| self.num_heads = num_heads | |
| self.head_dim = dim // num_heads | |
| self.q_proj = nn.Linear(dim, dim) | |
| self.k_proj = nn.Linear(dim, dim) | |
| self.v_proj = nn.Linear(dim, dim) | |
| self.out_proj = nn.Linear(dim, dim) | |
| # --- UPGRADE 1 CHANGE --- | |
| # Replace static nn.Parameter with a network to generate the bias | |
| self.bias_generator = nn.Sequential( | |
| nn.Linear(dim, dim // 2), | |
| nn.ReLU(), | |
| nn.Linear(dim // 2, self.num_heads * self.head_dim) | |
| ) | |
| # --- END UPGRADE 1 CHANGE --- | |
| def forward(self, x): | |
| B, T, D = x.shape | |
| # --- UPGRADE 1 CHANGE --- | |
| # Generate a sequence-specific dynamic bias | |
| global_context = x.mean(dim=1) # (B, D) | |
| dynamic_bias = self.bias_generator(global_context).view(B, self.num_heads, self.head_dim) | |
| # --- END UPGRADE 1 CHANGE --- | |
| Q = self.q_proj(x).view(B, T, self.num_heads, self.head_dim).transpose(1, 2) | |
| K = self.k_proj(x).view(B, T, self.num_heads, self.head_dim).transpose(1, 2) | |
| V = self.v_proj(x).view(B, T, self.num_heads, self.head_dim).transpose(1, 2) | |
| scores = torch.matmul(Q, K.transpose(-2, -1)) / np.sqrt(self.head_dim) | |
| # Apply the DYNAMIC Fractal Attentional Resonance | |
| bias = dynamic_bias.unsqueeze(-1) # Shape: (B, num_heads, head_dim, 1) | |
| fractal_resonance = torch.matmul(Q, bias) # Result shape: (B, num_heads, T, 1) | |
| scores = scores + fractal_resonance # Broadcasting applies the bias | |
| attn_weights = F.softmax(scores, dim=-1) | |
| context = torch.matmul(attn_weights, V) | |
| context = context.transpose(1, 2).contiguous().view(B, T, D) | |
| return self.out_proj(context) | |
| # All other modules from the original code remain the same... | |
| class FAR_TransformerEncoderLayer(nn.Module): | |
| def __init__(self, d_model, nhead, dim_feedforward, dropout=0.1, activation='gelu'): | |
| super().__init__() | |
| self.self_attn = FractalAttentionalResonance(d_model, nhead) # Uses the upgraded FAR | |
| self.linear1 = nn.Linear(d_model, dim_feedforward) | |
| self.dropout = nn.Dropout(dropout) | |
| self.linear2 = nn.Linear(dim_feedforward, d_model) | |
| self.norm1 = nn.LayerNorm(d_model) | |
| self.norm2 = nn.LayerNorm(d_model) | |
| self.dropout1 = nn.Dropout(dropout) | |
| self.dropout2 = nn.Dropout(dropout) | |
| self.activation = F.gelu if activation == 'gelu' else F.relu | |
| def forward(self, src): | |
| src2 = self.self_attn(src) | |
| src = src + self.dropout1(src2) | |
| src = self.norm1(src) | |
| src2 = self.linear2(self.dropout(self.activation(self.linear1(src)))) | |
| src = src + self.dropout2(src2) | |
| src = self.norm2(src) | |
| return src | |
| class CustomTransformerEncoder(nn.Module): | |
| def __init__(self, encoder_layer, num_layers): | |
| super().__init__() | |
| self.layers = nn.ModuleList([encoder_layer for _ in range(num_layers)]) | |
| self.num_layers = num_layers | |
| def forward(self, src): | |
| output = src | |
| for mod in self.layers: | |
| output = mod(output) | |
| return output | |
| # All other helper modules (FractalEmbeddingLayer, DynamicGating, etc.) are identical | |
| # to the original script and are included here for completeness. | |
| class FractalEmbeddingLayer(nn.Module): | |
| def __init__(self, dim, num_fractals=4): | |
| super().__init__() | |
| self.num_fractals = num_fractals | |
| self.dim = dim | |
| self.dims = nn.Parameter(torch.rand(num_fractals) * 2 + 1) | |
| self.weight_generator = nn.Sequential( | |
| nn.Linear(dim, dim // 2), | |
| nn.ReLU(), | |
| nn.Linear(dim // 2, num_fractals) | |
| ) | |
| self.fractal_functions = [ | |
| lambda x: torch.sin(x * 2 * math.pi), | |
| lambda x: x - torch.floor(x), | |
| lambda x: 4 * x * (1 - x), | |
| lambda x: torch.sigmoid(5 * (x - 0.5)) | |
| ] | |
| def forward(self, x): | |
| x_safe = torch.sigmoid(x) | |
| p_logits = self.weight_generator(x.mean(dim=1)) | |
| p_weights = F.softmax(p_logits, dim=-1).unsqueeze(1).unsqueeze(-1) | |
| fractal_outputs = [] | |
| for i in range(self.num_fractals): | |
| d = self.dims[i] | |
| x_powered = torch.pow(x_safe, 1.0 / d) | |
| f_i = self.fractal_functions[i] | |
| output = f_i(x_powered) | |
| fractal_outputs.append(output.unsqueeze(-2)) | |
| fractal_stack = torch.cat(fractal_outputs, dim=-2) | |
| weighted_fractals = p_weights * fractal_stack | |
| pfaf_embedding = torch.sum(weighted_fractals, dim=-2) | |
| return x + pfaf_embedding | |
| class DynamicGating(nn.Module): | |
| def __init__(self, dim, num_experts=4): | |
| super().__init__() | |
| self.gate_network = nn.Sequential(nn.Linear(dim, num_experts), nn.Softmax(dim=-1)) | |
| self.expert_networks = nn.ModuleList([nn.Linear(dim, dim) for _ in range(num_experts)]) | |
| def forward(self, x): | |
| gates = self.gate_network(x) | |
| expert_outputs = [expert(x) for expert in self.expert_networks] | |
| expert_stack = torch.stack(expert_outputs, dim=-1) | |
| gates = gates.unsqueeze(-2) | |
| return torch.sum(expert_stack * gates, dim=-1) | |
| class ContinuousResonanceField(nn.Module): | |
| def __init__(self, dim): | |
| super().__init__() | |
| self.field_mlp = nn.Sequential(nn.Linear(dim, dim), nn.GELU(), nn.LayerNorm(dim)) | |
| self.field_attention = nn.MultiheadAttention(dim, num_heads=4, batch_first=True) | |
| def forward(self, query, context): | |
| field_context = self.field_mlp(context) | |
| attended_field, _ = self.field_attention(query, field_context, field_context) | |
| return attended_field | |
| class QuantumInspiredFusion(nn.Module): | |
| def __init__(self, dim, num_qubits=4): | |
| super().__init__() | |
| self.num_qubits = num_qubits | |
| self.state_prep_know = nn.Linear(dim, num_qubits * 2) | |
| self.state_prep_exec = nn.Linear(dim, num_qubits * 2) | |
| self.entanglement_gate = nn.Parameter(torch.randn(num_qubits, num_qubits)) | |
| self.measurement = nn.Linear(num_qubits * 2, dim) | |
| self.norm = nn.LayerNorm(dim) | |
| def forward(self, know, exec): | |
| B, T, _ = know.shape | |
| know_state = self.state_prep_know(know).view(B, T, self.num_qubits, 2) | |
| exec_state = self.state_prep_exec(exec).view(B, T, self.num_qubits, 2) | |
| k_real, k_imag = know_state[..., 0], know_state[..., 1] | |
| e_real, e_imag = exec_state[..., 0], exec_state[..., 1] | |
| ent_real = k_real * e_real - k_imag * e_imag | |
| ent_imag = k_real * e_imag + k_imag * e_real | |
| ent_real = torch.matmul(ent_real, self.entanglement_gate) | |
| ent_imag = torch.matmul(ent_imag, self.entanglement_gate) | |
| state = torch.stack([ent_real, ent_imag], dim=-1).reshape(B, T, -1) | |
| return self.norm(self.measurement(state)) | |
| class HierarchicalMemorySystem(nn.Module): | |
| def __init__(self, dim, scales=[32, 16, 8]): | |
| super().__init__() | |
| self.memory_banks = nn.ParameterList([nn.Parameter(torch.randn(1, s, dim)) for s in scales]) | |
| self.projs = nn.ModuleList([nn.Linear(dim, dim) for _ in range(len(scales) * 3)]) | |
| self.gates = nn.ModuleList([nn.Linear(dim * 2, dim) for _ in scales]) | |
| self.scale_attention = nn.MultiheadAttention(dim, num_heads=4, batch_first=True) | |
| self.consolidation = nn.GRU(dim, dim, batch_first=True) | |
| def forward(self, x): | |
| B, T, D = x.shape | |
| scale_outputs = [] | |
| for i, memory in enumerate(self.memory_banks): | |
| mem = memory.expand(B, -1, -1) | |
| query, key, value = self.projs[i*3](x), self.projs[i*3+1](mem), self.projs[i*3+2](mem) | |
| scores = torch.bmm(query, key.transpose(1, 2)) / math.sqrt(D) | |
| attended_mem = torch.bmm(F.softmax(scores, dim=-1), value) | |
| gate = torch.sigmoid(self.gates[i](torch.cat([x, attended_mem], dim=-1))) | |
| scale_outputs.append(x * (1 - gate) + attended_mem * gate) | |
| fused_scales, _ = self.scale_attention(x, torch.cat(scale_outputs, dim=1), torch.cat(scale_outputs, dim=1)) | |
| consolidated, _ = self.consolidation(fused_scales) | |
| return consolidated | |
| class NeuralODE(nn.Module): | |
| def __init__(self, dim, num_steps=4): | |
| super().__init__() | |
| self.num_steps = num_steps | |
| self.ode_net = nn.Sequential(nn.Linear(dim, dim * 2), nn.Tanh(), nn.Linear(dim * 2, dim)) | |
| def forward(self, x): | |
| state = x | |
| dt = 1.0 / self.num_steps | |
| for _ in range(self.num_steps): | |
| state = state + dt * self.ode_net(state) | |
| return state | |
| # ============================================================================= | |
| # ZRIA-Ultimate Main Model with UPGRADES | |
| # ============================================================================= | |
| class ZRIA_Ultimate_PFAF(nn.Module): | |
| def __init__(self, dim, vocab_size, action_dim, max_seq_len=128): | |
| super().__init__() | |
| self.token_embedding = nn.Embedding(vocab_size, dim) | |
| self.positional_embedding = nn.Parameter(torch.randn(1, max_seq_len, dim)) | |
| self.modality_embedding = nn.Parameter(torch.randn(2, dim)) | |
| self.fractal_embedding = FractalEmbeddingLayer(dim) | |
| # Using the upgraded FAR encoder | |
| far_encoder_layer = FAR_TransformerEncoderLayer(d_model=dim, nhead=4, dim_feedforward=dim * 2) | |
| self.knowledge_encoder = CustomTransformerEncoder(far_encoder_layer, num_layers=2) | |
| self.execution_encoder = CustomTransformerEncoder(far_encoder_layer, num_layers=2) | |
| # --- UPGRADE 2 CHANGE --- | |
| # Add cross-attention modules for path communication | |
| self.know_to_exec_cross_attn = nn.MultiheadAttention(dim, num_heads=4, batch_first=True) | |
| self.exec_to_know_cross_attn = nn.MultiheadAttention(dim, num_heads=4, batch_first=True) | |
| self.cross_norm_k = nn.LayerNorm(dim) | |
| self.cross_norm_e = nn.LayerNorm(dim) | |
| # --- END UPGRADE 2 CHANGE --- | |
| self.dynamic_gating = DynamicGating(dim) | |
| self.resonance_field = ContinuousResonanceField(dim) | |
| self.quantum_fusion = QuantumInspiredFusion(dim) | |
| self.hierarchical_memory = HierarchicalMemorySystem(dim) | |
| self.neural_ode = NeuralODE(dim) | |
| self.fusion_norm_1 = nn.LayerNorm(dim) | |
| self.fusion_norm_2 = nn.LayerNorm(dim) | |
| self.output_projection = nn.Linear(dim, dim) | |
| self.output_router = nn.Linear(dim, 2) | |
| self.text_head = nn.Linear(dim, 3) | |
| self.action_head = nn.Sequential(nn.Linear(dim, dim // 2), nn.GELU(), nn.Linear(dim // 2, action_dim)) | |
| def forward(self, input_ids, modality_idx, mode='auto'): | |
| B, T = input_ids.shape | |
| x = self.token_embedding(input_ids) + self.positional_embedding[:, :T, :] + self.modality_embedding[modality_idx].unsqueeze(1) | |
| pfaf_x = self.fractal_embedding(x) | |
| know_path = self.knowledge_encoder(pfaf_x) | |
| exec_path = self.execution_encoder(pfaf_x) | |
| # --- UPGRADE 2 CHANGE --- | |
| # Allow paths to communicate via cross-attention | |
| exec_from_know, _ = self.know_to_exec_cross_attn(query=exec_path, key=know_path, value=know_path) | |
| exec_path = self.cross_norm_e(exec_path + exec_from_know) | |
| know_from_exec, _ = self.exec_to_know_cross_attn(query=know_path, key=exec_path, value=exec_path) | |
| know_path = self.cross_norm_k(know_path + know_from_exec) | |
| # --- END UPGRADE 2 CHANGE --- | |
| base = know_path + exec_path | |
| fused = self.dynamic_gating(base) + base | |
| fused = self.resonance_field(fused, base) + fused | |
| fused = self.quantum_fusion(fused, base) + fused | |
| fused = self.fusion_norm_1(fused) | |
| mem = self.hierarchical_memory(fused) + fused | |
| ode = self.neural_ode(mem) + mem | |
| final_repr = self.output_projection(self.fusion_norm_2(ode))[:, -1, :] | |
| if mode == 'auto': | |
| route_probs = F.softmax(self.output_router(final_repr), dim=-1) | |
| return {'text_logits': self.text_head(final_repr), 'action_output': self.action_head(final_repr), 'route_probs': route_probs} | |
| elif mode == 'text': | |
| return self.text_head(final_repr) | |
| else: | |
| return self.action_head(final_repr) | |
| # ============================================================================= | |
| # Data Handling, Training, and Inference (Unchanged) | |
| # ============================================================================= | |
| class AdvancedTokenizer: | |
| def __init__(self, corpus): | |
| special_tokens = ['<PAD>', '<UNK>', '<KNOW>', '<DO>'] | |
| all_chars = sorted(list(set("".join(corpus)))) | |
| self.char_to_idx = {t: i for i, t in enumerate(special_tokens)} | |
| for char in all_chars: | |
| if char not in self.char_to_idx: self.char_to_idx[char] = len(self.char_to_idx) | |
| self.vocab_size = len(self.char_to_idx) | |
| self.pad_idx = self.char_to_idx['<PAD>'] | |
| def encode(self, text, prefix): | |
| tokens = [self.char_to_idx['<KNOW>' if prefix == 'KNOW' else '<DO>']] | |
| for char in text.lower(): tokens.append(self.char_to_idx.get(char, self.char_to_idx['<UNK>'])) | |
| return tokens | |
| class EnhancedDataset(Dataset): | |
| def __init__(self, data, tokenizer, max_len=64): | |
| self.data, self.tokenizer, self.max_len = data, tokenizer, max_len | |
| def __len__(self): return len(self.data) | |
| def __getitem__(self, idx): | |
| prefix, text, label = self.data[idx] | |
| tokens = self.tokenizer.encode(text, prefix) | |
| tokens += [self.tokenizer.pad_idx] * (self.max_len - len(tokens)) | |
| return {'input_ids': torch.tensor(tokens[:self.max_len], dtype=torch.long), | |
| 'label': torch.tensor(label, dtype=torch.float32), | |
| 'mode': 'text' if prefix == 'KNOW' else 'action', | |
| 'modality_idx': torch.tensor(0 if prefix == 'KNOW' else 1, dtype=torch.long)} | |
| def advanced_train_loop(model, dataloader, optimizer, scheduler, epochs=150, device='cpu'): | |
| model.to(device) | |
| model.train() | |
| text_crit, action_crit, route_crit = nn.CrossEntropyLoss(), nn.MSELoss(), nn.CrossEntropyLoss() | |
| print("=== Starting ZRIA Training (Upgraded Architecture) ===") | |
| for epoch in range(epochs): | |
| total_loss = 0 | |
| for batch in dataloader: | |
| ids, labels, mod_idx = batch['input_ids'].to(device), batch['label'].to(device), batch['modality_idx'].to(device) | |
| optimizer.zero_grad() | |
| outputs = model(ids, mod_idx, mode='auto') | |
| text_mask = torch.tensor([m == 'text' for m in batch['mode']], device=device) | |
| action_mask = ~text_mask | |
| loss = 0 | |
| if text_mask.any(): | |
| loss += text_crit(outputs['text_logits'][text_mask], labels[text_mask].long()) | |
| loss += 0.1 * route_crit(outputs['route_probs'][text_mask], torch.zeros(text_mask.sum(), dtype=torch.long, device=device)) | |
| if action_mask.any(): | |
| loss += 0.1 * action_crit(outputs['action_output'][action_mask].squeeze(-1), labels[action_mask]) | |
| loss += 0.1 * route_crit(outputs['route_probs'][action_mask], torch.ones(action_mask.sum(), dtype=torch.long, device=device)) | |
| if isinstance(loss, torch.Tensor): | |
| loss.backward() | |
| torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0) | |
| optimizer.step() | |
| total_loss += loss.item() | |
| scheduler.step() | |
| if (epoch + 1) % 10 == 0 or epoch == epochs - 1: | |
| print(f"Epoch {epoch+1}/{epochs} | Avg Loss: {total_loss/len(dataloader):.4f} | LR: {scheduler.get_last_lr()[0]:.6f}") | |
| print("=== Training Complete ===") | |
| def enhanced_inference(model, tokenizer, test_cases, device='cpu'): | |
| model.to(device).eval() | |
| print("\n=== Enhanced Inference Results (Upgraded Architecture) ===") | |
| with torch.no_grad(): | |
| for prefix, text, expected in test_cases: | |
| tokens = tokenizer.encode(text, prefix) | |
| tokens += [tokenizer.pad_idx] * (64 - len(tokens)) | |
| ids = torch.tensor([tokens], dtype=torch.long).to(device) | |
| mod_idx = torch.tensor([0 if prefix == 'KNOW' else 1], dtype=torch.long).to(device) | |
| outputs = model(ids, mod_idx, mode='auto') | |
| probs = outputs['route_probs'][0] | |
| print(f"\nInput: '{prefix}: {text}' | Expected: {expected}") | |
| print(f"Routing -> Text: {probs[0]:.3f}, Action: {probs[1]:.3f}") | |
| if probs[0] > probs[1]: | |
| pred = torch.argmax(outputs['text_logits'][0]).item() | |
| print(f"Prediction: Routed to KNOWLEDGE -> {['Negative', 'Neutral', 'Positive'][pred]}") | |
| else: | |
| pred = outputs['action_output'][0].item() | |
| print(f"Prediction: Routed to EXECUTION -> Vowel Count: {pred:.2f}") | |
| # ============================================================================= | |
| # Main Execution Block | |
| # ============================================================================= | |
| if __name__ == '__main__': | |
| device = 'cuda' if torch.cuda.is_available() else 'cpu' | |
| print(f"Using device: {device}") | |
| enhanced_data = [ | |
| ('KNOW', 'i absolutely love this amazing product', 2), ('KNOW', 'this is the best thing ever created', 2), | |
| ('KNOW', 'wonderful experience with great service', 2), ('KNOW', 'a truly fantastic and brilliant idea', 2), | |
| ('KNOW', 'excellent work, very impressive', 2), ('KNOW', 'i am so happy with the results', 2), | |
| ('KNOW', 'i hate this terrible awful thing', 0), ('KNOW', 'horrible experience with bad service', 0), | |
| ('KNOW', 'a disappointing and frustrating product', 0), ('KNOW', 'this is the worst i have ever seen', 0), | |
| ('KNOW', 'i am very unhappy and angry', 0), ('KNOW', 'a complete failure and a waste of time', 0), | |
| ('KNOW', 'the meeting is scheduled for tomorrow', 1), ('KNOW', 'this is a factual statement', 1), | |
| ('KNOW', 'the document is on the table', 1), ('KNOW', 'standard operating procedure', 1), | |
| ('DO', 'how many vowels in this sentence', 8), ('DO', 'count all the vowels here please', 9), | |
| ('DO', 'the quick brown fox jumps over the lazy dog', 11), ('DO', 'artificial intelligence', 7), | |
| ('DO', 'rhythm and blues', 3), ('DO', 'fly by', 1), ('DO', 'aeiou', 5), | |
| ] * 10 | |
| random.shuffle(enhanced_data) | |
| corpus = [text for _, text, _ in enhanced_data] | |
| tokenizer = AdvancedTokenizer(corpus) | |
| dataset = EnhancedDataset(enhanced_data, tokenizer, max_len=64) | |
| dataloader = DataLoader(dataset, batch_size=16, shuffle=True) | |
| model = ZRIA_Ultimate_PFAF(dim=128, vocab_size=tokenizer.vocab_size, action_dim=1, max_seq_len=64) | |
| total_params = sum(p.numel() for p in model.parameters() if p.requires_grad) | |
| print(f"Model Architecture: ZRIA-Ultimate (Upgraded)") | |
| print(f"Trainable parameters: {total_params:,} (~{total_params * 4 / 1e6:.1f}MB)") | |
| optimizer = optim.AdamW(model.parameters(), lr=3e-4, weight_decay=0.01) | |
| scheduler = CosineAnnealingWarmRestarts(optimizer, T_0=30, T_mult=2, eta_min=1e-6) | |
| advanced_train_loop(model, dataloader, optimizer, scheduler, epochs=150, device=device) | |
| test_cases = [ | |
| ("KNOW", "this is a wonderful experience", "Positive"), | |
| ("KNOW", "i hate this terrible product", "Negative"), | |
| ("KNOW", "the report is due on friday", "Neutral"), | |
| ("DO", "count the vowels in this example", 8), | |
| ("DO", "a quick test", 3), | |
| ("DO", "why", 0), | |
| ] | |
| enhanced_inference(model, tokenizer, test_cases, device=device) |