""" Rete neurale su mesh geodetica sferica (PyTorch). - TriangleBlock: pesi locali + router ternario (3 lati) - CausalAttnBlock: self-attention causale + FF (contesto temporale) - SphericalMeshModel: emb → CausalAttn × N → walk mesh batched → LM head - Pesi faccia impacchettati a coppie (pair packing) per buffer contigui GEMM-friendly """ from __future__ import annotations from dataclasses import dataclass from typing import List, Optional, Tuple import math import numpy as np import torch import torch.nn as nn import torch.nn.functional as F from mesh_topology import GeodesicMesh, build_geodesic_mesh, nearest_face @dataclass class WalkConfig: max_hops: int = 8 # Se la probabilità del “exit” (soft) supera la soglia, si ferma prima exit_threshold: float = 0.55 # Temperatura del router ternario router_temperature: float = 1.0 # Se True, usa argmax (discreto); altrimenti Gumbel-Softmax straight-through hard_routing: bool = True class TriangleBlock(nn.Module): """Micro-blocco su una faccia: MLP locale + logits verso 3 lati + exit.""" def __init__(self, d_model: int, d_hidden: Optional[int] = None): super().__init__() h = d_hidden or d_model self.norm = nn.LayerNorm(d_model) self.ff = nn.Sequential( nn.Linear(d_model, h), nn.GELU(), nn.Linear(h, d_model), ) # 3 lati + 1 “esci / emetti” self.router = nn.Linear(d_model, 4) def forward(self, x: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]: """ x: (..., d) returns: (x_out, router_logits) con logits shape (..., 4) """ h = self.ff(self.norm(x)) x_out = x + h logits = self.router(x_out) return x_out, logits class PairedFaceBank(nn.Module): """ Banca pesi: per ogni coppia di facce adiacenti memorizza 2 TriangleBlock in un modulo ModuleList, con indici face→block per lookup O(1). I parametri di ogni coppia restano contigui nel ModuleList (ordine packing). """ def __init__(self, mesh: GeodesicMesh, d_model: int, d_hidden: Optional[int] = None): super().__init__() self.num_faces = mesh.num_faces self.register_buffer("adjacency", torch.from_numpy(mesh.adjacency.astype(np.int64))) self.register_buffer("pairs", torch.from_numpy(mesh.pairs.astype(np.int64))) # face_id → index nel ModuleList `blocks` face_to_slot = np.full(mesh.num_faces, -1, dtype=np.int64) blocks: List[TriangleBlock] = [] for fa, fb in mesh.pairs: face_to_slot[int(fa)] = len(blocks) blocks.append(TriangleBlock(d_model, d_hidden)) face_to_slot[int(fb)] = len(blocks) blocks.append(TriangleBlock(d_model, d_hidden)) for u in mesh.unpaired: face_to_slot[int(u)] = len(blocks) blocks.append(TriangleBlock(d_model, d_hidden)) self.blocks = nn.ModuleList(blocks) self.register_buffer("face_to_slot", torch.from_numpy(face_to_slot)) def forward_face(self, face_id: int, x: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]: slot = int(self.face_to_slot[face_id].item()) return self.blocks[slot](x) class CausalSelfAttention(nn.Module): """Multi-head self-attention con maschera causale (lookahead vietato).""" def __init__(self, d_model: int, n_heads: int, dropout: float = 0.0): super().__init__() if d_model % n_heads != 0: raise ValueError(f"d_model={d_model} non divisibile per n_heads={n_heads}") self.d_model = d_model self.n_heads = n_heads self.head_dim = d_model // n_heads self.qkv = nn.Linear(d_model, 3 * d_model, bias=False) self.proj = nn.Linear(d_model, d_model, bias=False) self.attn_drop = nn.Dropout(dropout) self.resid_drop = nn.Dropout(dropout) def forward(self, x: torch.Tensor) -> torch.Tensor: # x: (B, T, d) b, t, _ = x.shape qkv = self.qkv(x).view(b, t, 3, self.n_heads, self.head_dim) qkv = qkv.permute(2, 0, 3, 1, 4) # (3, B, H, T, hd) q, k, v = qkv[0], qkv[1], qkv[2] scale = 1.0 / math.sqrt(self.head_dim) # PyTorch SDPA con is_causal=True (Flash/mem-efficient quando disponibili) y = F.scaled_dot_product_attention( q, k, v, attn_mask=None, dropout_p=self.attn_drop.p if self.training else 0.0, is_causal=True, scale=scale, ) # (B, H, T, hd) y = y.transpose(1, 2).contiguous().view(b, t, self.d_model) return self.resid_drop(self.proj(y)) class CausalAttnBlock(nn.Module): """Pre-norm: LayerNorm → causal attn → residual → LayerNorm → FF → residual.""" def __init__( self, d_model: int, n_heads: int, d_ff: Optional[int] = None, dropout: float = 0.0, ): super().__init__() h = d_ff or (4 * d_model) self.ln1 = nn.LayerNorm(d_model) self.attn = CausalSelfAttention(d_model, n_heads, dropout=dropout) self.ln2 = nn.LayerNorm(d_model) self.ff = nn.Sequential( nn.Linear(d_model, h), nn.GELU(), nn.Linear(h, d_model), nn.Dropout(dropout), ) def forward(self, x: torch.Tensor) -> torch.Tensor: x = x + self.attn(self.ln1(x)) x = x + self.ff(self.ln2(x)) return x def _default_n_heads(d_model: int) -> int: for h in (8, 4, 2, 1): if d_model % h == 0: return h return 1 class SphericalMeshModel(nn.Module): """ LM: token → emb (+pos) → CausalAttn × N → walk mesh parallelo (B*T) → vocab. Contesto temporale via attenzione causale (no carry sequenziale). """ def __init__( self, vocab_size: int, d_model: int = 64, frequency: int = 2, walk: Optional[WalkConfig] = None, d_hidden: Optional[int] = None, n_attn_layers: int = 2, n_heads: Optional[int] = None, max_seq_len: int = 512, use_mesh_walk: bool = True, attn_dropout: float = 0.0, compass_spiral: bool = False, ): super().__init__() self.compass_spiral = bool(compass_spiral) self.mesh: GeodesicMesh = build_geodesic_mesh( frequency, compass_spiral=self.compass_spiral ) self.walk = walk or WalkConfig() self.d_model = d_model self.vocab_size = vocab_size self.n_attn_layers = int(n_attn_layers) self.use_mesh_walk = bool(use_mesh_walk) self.max_seq_len = int(max_seq_len) heads = int(n_heads) if n_heads is not None else _default_n_heads(d_model) self.tok_emb = nn.Embedding(vocab_size, d_model) nn.init.normal_(self.tok_emb.weight, mean=0.0, std=0.02) self.pos_emb = nn.Embedding(self.max_seq_len, d_model) nn.init.normal_(self.pos_emb.weight, mean=0.0, std=0.02) self.attn_blocks = nn.ModuleList( [ CausalAttnBlock(d_model, heads, d_ff=d_hidden or (4 * d_model), dropout=attn_dropout) for _ in range(self.n_attn_layers) ] ) # Proietta embedding → direzione 3D sulla sfera (faccia di ingresso) self.to_sphere = nn.Linear(d_model, 3) self.bank = PairedFaceBank(self.mesh, d_model, d_hidden) self.final_norm = nn.LayerNorm(d_model) self.lm_head = nn.Linear(d_model, vocab_size, bias=False) # embedding tied (come goTommy2) self.lm_head.weight = self.tok_emb.weight self.emb_scale = d_model**-0.5 # Buffer geometria per nearest face su GPU/CPU self.register_buffer( "centroids", torch.from_numpy(self.mesh.face_centroids.astype(np.float32)), ) self.register_buffer( "adjacency", torch.from_numpy(self.mesh.adjacency.astype(np.int64)), ) def project_to_face(self, h: torch.Tensor) -> torch.Tensor: """h (..., d) → face ids (...)""" direction = self.to_sphere(h) direction = F.normalize(direction, dim=-1) # (..., F) scores = direction @ self.centroids.T return scores.argmax(dim=-1) def _route_step( self, face: torch.Tensor, logits: torch.Tensor, ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]: """ face: (B,) long — faccia corrente logits: (B, 4) — 3 lati + exit returns: next_face (B,), exit_mask (B,) bool, side_choice (B,) in {0,1,2,-1} """ temp = max(self.walk.router_temperature, 1e-6) probs = F.softmax(logits / temp, dim=-1) exit_prob = probs[:, 3] side_probs = probs[:, :3] if self.walk.hard_routing: side = side_probs.argmax(dim=-1) else: # Gumbel-softmax straight-through sui 3 lati side = F.gumbel_softmax(logits[:, :3] / temp, hard=True, dim=-1).argmax(dim=-1) exit_mask = exit_prob >= self.walk.exit_threshold # lookup vettoriale: adjacency[face, side] next_face = self.adjacency[face.long(), side.long()] side_out = torch.where(exit_mask, torch.full_like(side, -1), side) return next_face, exit_mask, side_out def walk_token( self, h: torch.Tensor, start_face: Optional[torch.Tensor] = None, record_path: bool = False, ) -> Tuple[torch.Tensor, Optional[List[List[int]]]]: """ h: (B, d) stato iniziale token — tipicamente B*T flattened returns: h_final (B, d), paths opzionale list per batch """ b = h.shape[0] face = start_face if start_face is not None else self.project_to_face(h) paths: Optional[List[List[int]]] = [[int(face[i].item())] for i in range(b)] if record_path else None active = torch.ones(b, dtype=torch.bool, device=h.device) for _ in range(self.walk.max_hops): if not active.any(): break # Esegue i blocchi per-faccia (batch misto: loop su facce uniche) logits = h.new_zeros(b, 4) h_new = h.clone() unique_faces = face[active].unique() for f_id in unique_faces.tolist(): mask = active & (face == f_id) if not mask.any(): continue out, lg = self.bank.forward_face(int(f_id), h[mask]) h_new[mask] = out logits[mask] = lg h = h_new next_face, exit_mask, _ = self._route_step(face, logits) # Solo i token ancora attivi possono uscire / muoversi moved = active & ~exit_mask face = torch.where(moved, next_face, face) just_exit = active & exit_mask active = active & ~exit_mask if record_path and paths is not None: for i in range(b): if moved[i]: paths[i].append(int(face[i].item())) elif just_exit[i]: paths[i].append(-1) # sentinel exit return h, paths def forward( self, input_ids: torch.Tensor, record_path: bool = False, ) -> Tuple[torch.Tensor, Optional[List[List[List[int]]]]]: """ input_ids: (B, T) returns: logits (B, T, V), paths [B][T][hops] se richiesto Pipeline: emb(+pos) → CausalAttnBlock × N → walk_token(B*T) → final_norm → lm_head """ b, t = input_ids.shape if t > self.max_seq_len: raise ValueError(f"seq_len={t} > max_seq_len={self.max_seq_len}") pos = torch.arange(t, device=input_ids.device).unsqueeze(0) # (1, T) h = self.tok_emb(input_ids) * self.emb_scale + self.pos_emb(pos) for block in self.attn_blocks: h = block(h) # (B, T, d) paths: Optional[List[List[List[int]]]] = None if self.use_mesh_walk: flat = h.reshape(b * t, self.d_model) flat_out, paths_flat = self.walk_token(flat, record_path=record_path) h = flat_out.view(b, t, self.d_model) if record_path and paths_flat is not None: # paths_flat[bi*T + ti] → paths[bi][ti] paths = [ [paths_flat[bi * t + ti] for ti in range(t)] for bi in range(b) ] h = self.final_norm(h) logits = self.lm_head(h) # soft-cap: evita CE esplosa con residual walk + tied emb logits = 30.0 * torch.tanh(logits / 30.0) return logits, paths def count_parameters(model: nn.Module) -> int: return sum(p.numel() for p in model.parameters() if p.requires_grad)