V6.5-V2-dynamic: upload batch (scripts + state + model) [16 files]

attention_multimodal.py

@@ -0,0 +1,114 @@
+"""
+Xavante - attention_multimodal.py
+Responsabilidade: Multi-head attention multimodal (Teorema 11.1/11.2).
+Suporta attention entre modalidades e dentro de modalidade.
+"""
+from __future__ import annotations
+
+import logging
+import math
+from typing import Optional
+
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+
+logger = logging.getLogger(__name__)
+
+
+class MultiHeadAttention(nn.Module):
+    """
+    MHA padrão com suporte a:
+    - Mascara causal
+    - Mascara multimodal (mod crossing)
+    - Attention 2M tokens via chunked attention (Teorema 6.1)
+    """
+
+    def __init__(
+        self,
+        d_model: int,
+        n_heads: int = 8,
+        dropout: float = 0.0,
+        max_chunk: int = 4096,
+    ):
+        super().__init__()
+        assert d_model % n_heads == 0
+        self.d_model = d_model
+        self.n_heads = n_heads
+        self.d_head = d_model // n_heads
+        self.max_chunk = max_chunk
+        self.qkv = nn.Linear(d_model, 3 * d_model, bias=True)
+        self.out = nn.Linear(d_model, d_model)
+        self.dropout = nn.Dropout(dropout)
+
+    def forward(
+        self,
+        x: torch.Tensor,
+        mask: Optional[torch.Tensor] = None,
+        kv: Optional[torch.Tensor] = None,
+    ) -> torch.Tensor:
+        B, L, D = x.shape
+        if kv is None:
+            qkv = self.qkv(x)
+            q, k, v = qkv.chunk(3, dim=-1)
+        else:
+            q = self.qkv(x)[:, :, :D]
+            kv_proj = self.qkv(kv)
+            k = kv_proj[:, :, D : 2 * D]
+            v = kv_proj[:, :, 2 * D :]
+        # reshape para heads
+        q = q.view(B, L, self.n_heads, self.d_head).transpose(1, 2)
+        k = k.view(B, -1, self.n_heads, self.d_head).transpose(1, 2)
+        v = v.view(B, -1, self.n_heads, self.d_head).transpose(1, 2)
+
+        # Chunked attention para sequencias longas (Teorema 6.1)
+        if L > self.max_chunk:
+            # Normalize mask to 4D for chunked path
+            if mask is not None and mask.dim() == 2:
+                mask = mask.unsqueeze(0).unsqueeze(0)
+            elif mask is not None and mask.dim() == 3:
+                mask = mask.unsqueeze(1)
+            return self._chunked_attention(q, k, v, mask)
+
+        scale = 1.0 / math.sqrt(self.d_head)
+        attn = (q @ k.transpose(-2, -1)) * scale  # [B, H, L, L]
+        if mask is not None:
+            attn = attn.masked_fill(mask == 0, float("-inf"))
+        attn = F.softmax(attn, dim=-1)
+        attn = self.dropout(attn)
+        out = attn @ v  # [B, H, L, d_head]
+        out = out.transpose(1, 2).contiguous().view(B, L, D)
+        return self.out(out)
+
+    def _chunked_attention(
+        self,
+        q: torch.Tensor,
+        k: torch.Tensor,
+        v: torch.Tensor,
+        mask: Optional[torch.Tensor],
+    ) -> torch.Tensor:
+        """Attention em blocos para janelas de 2M tokens (memoria controlada)."""
+        B, H, L, d = q.shape
+        Lk = k.shape[2]
+        chunk = self.max_chunk
+        outs = []
+        scale = 1.0 / math.sqrt(d)
+        for i in range(0, L, chunk):
+            qi = q[:, :, i : i + chunk]
+            out_chunk = []
+            for j in range(0, Lk, chunk):
+                kj = k[:, :, j : j + chunk]
+                vj = v[:, :, j : j + chunk]
+                attn = (qi @ kj.transpose(-2, -1)) * scale
+                if mask is not None:
+                    m_chunk = mask[:, :, i : i + chunk, j : j + chunk]
+                    attn = attn.masked_fill(m_chunk == 0, float("-inf"))
+                attn = F.softmax(attn, dim=-1)
+                out_chunk.append(attn @ vj)
+            outs.append(torch.cat(out_chunk, dim=2))
+        out = torch.cat(outs, dim=2)
+        out = out.transpose(1, 2).contiguous().view(B, L, self.n_heads * d)
+        return self.out(out)
+
+
+__all__ = ["MultiHeadAttention"]

hyp_t.py

@@ -0,0 +1,100 @@
+"""hyp_t.py — HypT: cabeça de hipótese (correção delta). Implementa o Lema 3.
+
+A Hipótese é treinada para cancelar o ruído de quantização W8A8:
+    y_final = y_hat + delta
+onde delta = hyp_T(o).
+
+Diferenças vs. TrainT:
+- Recebe `o` opcionalmente destacado (stop_grad) para isolar o sinal de
+  correção do gradiente principal
+- Mesma arquitetura (TransformerEncoder) — apenas o tratamento de gradiente
+  difere
+
+Reaproveita a filosofia do HypothesisNetwork do v13.9.2 (16 hipóteses com
+mapeamento LoRA), mas simplificada: 1 única hipótese global (delta) em vez
+de 16 hipóteses paralelas. A multiplexação é feita pelo OrqCell + Lema 1.
+
+============================================================================
+V6.4 — pgvector_lookup REMOVIDO (NÃO É MAIS NECESSÁRIO)
+============================================================================
+Conforme directive do usuário:
+  "pgvector_lookup não é mais necessário pela lógica do script seguinte"
+
+O KohonenLearningSystem (kohonen_refactored/kohonen_learning_system.py)
+contém o método find_bmu que realiza busca nearest-neighbor sobre o grid
+4D do SOM (864 neurônios para grid (6,6,6,4)), substituindo qualquer
+lookup pgvector externo. A decisão de aplicar punição é delegada ao
+protocolo interno do KohonenLearningSystem (1ª punição → activate_hypothesis,
+2ª punição → set_ewc_reference + reset).
+
+Consequentemente, HypT volta ao seu papel original: gerar a correção
+delta = hyp_T(o) sem consultar mapa externo. A integração com o SOM é
+feita no nível do trainer (KohonenLearningSystem.process_batch), não no
+nível da hipótese.
+
+Os contadores de punição (n_complete_skip, n_incomplete_apply, etc.)
+foram removidos pois não há mais decisão de skip baseada em pgvector.
+"""
+from __future__ import annotations
+
+import torch
+import torch.nn as nn
+
+
+class HypT(nn.Module):
+    """Transformer de hipótese: gera correção delta.
+
+    Args:
+        d_input: dimensão de entrada (saída do OrqCell = d_cache)
+        d_model: dimensão interna
+        nhead: nº de cabeças
+        d_ff: dimensão do FFN
+        output_dim: dimensão de saída (igual ao TrainT)
+        num_layers: nº de camadas
+        dropout: probabilidade de dropout
+
+    Forward:
+        o: (batch, d_input), stop_grad: bool → delta: (batch, output_dim)
+
+    Se stop_grad=True, aplica o.detach() no início — Lema 3 (isolar correção
+    do gradiente principal).
+    """
+
+    def __init__(
+        self,
+        d_input: int,
+        d_model: int,
+        nhead: int,
+        d_ff: int,
+        output_dim: int,
+        num_layers: int = 2,
+        dropout: float = 0.1,
+    ):
+        super().__init__()
+        self.d_input = d_input
+        self.d_model = d_model
+        self.output_dim = output_dim
+
+        self.proj_in = nn.Linear(d_input, d_model)
+        encoder_layer = nn.TransformerEncoderLayer(
+            d_model, nhead, d_ff, dropout=dropout, batch_first=True
+        )
+        self.encoder = nn.TransformerEncoder(encoder_layer, num_layers)
+        self.proj_out = nn.Linear(d_model, output_dim)
+
+    def forward(self, o: torch.Tensor, stop_grad: bool = True) -> torch.Tensor:
+        """o: (batch, d_input) → delta: (batch, output_dim)
+
+        V6.4: SEM consulta pgvector — delta é computado diretamente.
+        A decisão de aplicar ou não a correção é delegada ao
+        KohonenLearningSystem (que usa find_bmu internamente).
+        """
+        # Lema 3: stop_grad_hyp isola o sinal de correção
+        if stop_grad:
+            o = o.detach()
+
+        x = o.unsqueeze(1)               # (batch, 1, d_input)
+        x = self.proj_in(x)              # (batch, 1, d_model)
+        x = self.encoder(x)              # (batch, 1, d_model)
+        delta = self.proj_out(x.squeeze(1))  # (batch, output_dim)
+        return delta

som_metrics.py

@@ -0,0 +1,920 @@
+"""som_metrics.py — V6.5-V2-metrics: Métricas de aprendizado e indicadores de
+falha para Mapas Auto-Organizáveis (SOM / Redes de Kohonen).
+
+============================================================================
+MÉTRICAS PRINCIPAIS DE APRENDIZADO
+============================================================================
+
+1.1. Erro de Quantização (Quantization Error - QE)
+     Média das distâncias euclidianas entre cada vetor de entrada e o vetor
+     de pesos de seu neurônio vencedor (BMU). Mede a resolução ou ajuste fino
+     do mapa aos dados.
+
+         QE = (1/N) * Σ_i ||x_i - W_BMU(x_i)||₂
+
+1.2. Erro Topológico (Topological Error - TE)
+     Percentual de vetores de entrada cujos primeiro e segundo neurônios
+     mais próximos (BMU e segunda BMU) NÃO são vizinhos diretos na grade
+     de saída. Avalia preservação das relações geométricas.
+
+         TE = (1/N) * Σ_i 𝟙[¬adjacent(BMU_1(x_i), BMU_2(x_i))]
+
+1.3. Erro de Kaski-Lagus
+     Combinação ponderada que penaliza tanto a falta de resolução do vetor
+     de quantização quanto as quebras na continuidade topológica.
+
+         KL = α * QE_norm + β * TE
+         onde α=0.5, β=0.5 (pesos padrão)
+
+     Variante implementada: KL = QE * (1 + TE)  (penaliza QE quando TE é alto)
+
+1.4. Variância Explicada (Share of Explained Variance)
+     Fração da variabilidade total dos dados de entrada que é mapeada e
+     representada pelos vetores de referência (pesos) dos nós da rede.
+
+         VarExplained = 1 - Var(residual) / Var(total)
+         onde residual = x_i - W_BMU(x_i)
+
+============================================================================
+INDICADORES DE NÃO APRENDIZADO OU FALHA
+============================================================================
+
+2.1. Colapso Topológico (Topological Collapse)
+     Ocorre quando todos os vetores de peso convergem para o mesmo ponto
+     ou para uma linha reta. Indica perda total da dimensionalidade útil.
+
+     Detecção:
+       - std(weights) por dimensão < threshold (colapso total)
+       - rank(weights reshaped) < 2 (todos num ponto)
+       - rank < 3 (todos numa linha)
+
+2.2. Taxa de Neurônios Mortos (Dead Neurons)
+     Percentual de neurônios que NUNCA se tornam BMU para nenhum vetor.
+
+         DeadRate = (n_neurons - n_active_neurons) / n_neurons
+
+     Threshold: DeadRate > 0.5 indica problema.
+
+2.3. Estagnação do Erro de Quantização
+     QE estabiliza em patamar elevado ao longo das épocas. A taxa de
+     aprendizado decaiu rápido demais antes da fase de ordenação global.
+
+     Detecção: slope(QE) ≈ 0 E QE atual > 0.7 * max(QE histórico)
+
+2.4. Cruzamento de Vizinhança (Neighborhood Crossing)
+     Crescimento repentino ou oscilação crônica do TE nas épocas finais.
+
+     Detecção:
+       - std(TE_final_5_epochs) > 2 * std(TE_initial_5_epochs) (oscilação)
+       - TE[-1] > TE[-5] (crescimento no final)
+
+============================================================================
+AUTOR: V6.5-V2-metrics
+============================================================================
+"""
+from __future__ import annotations
+
+import math
+import logging
+from collections import defaultdict, deque
+from dataclasses import dataclass, field
+from typing import Any, Dict, List, Optional, Tuple
+
+import torch
+import torch.nn.functional as F
+
+logger = logging.getLogger(__name__)
+
+
+# ============================================================================
+# Dataclass para armazenar histórico de métricas
+# ============================================================================
+@dataclass
+class SOMMetricHistory:
+    """Histórico deslizante de métricas SOM para detecção de estagnação e
+    cruzamento de vizinhança."""
+    qe_history: deque = field(default_factory=lambda: deque(maxlen=50))
+    te_history: deque = field(default_factory=lambda: deque(maxlen=50))
+    kl_history: deque = field(default_factory=lambda: deque(maxlen=50))
+    var_explained_history: deque = field(default_factory=lambda: deque(maxlen=50))
+    dead_neuron_rate_history: deque = field(default_factory=lambda: deque(maxlen=50))
+    epoch: int = 0
+
+    def record(self, metrics: Dict[str, Any]) -> None:
+        """Registra métricas de uma época.
+
+        Nota: `dead_neuron_rate` pode ser float (valor direto) ou Dict
+        (saída de compute_all_metrics, com chave 'dead_neuron_rate').
+        """
+        self.qe_history.append(float(metrics.get("quantization_error", 0.0)))
+        self.te_history.append(float(metrics.get("topological_error", 0.0)))
+        self.kl_history.append(float(metrics.get("kaski_lagus_error", 0.0)))
+        self.var_explained_history.append(
+            float(metrics.get("explained_variance_share", 0.0))
+        )
+        # dead_neuron_rate pode ser float ou Dict
+        dnr = metrics.get("dead_neuron_rate", 0.0)
+        if isinstance(dnr, dict):
+            dnr_val = float(dnr.get("dead_neuron_rate", 0.0))
+        else:
+            dnr_val = float(dnr)
+        self.dead_neuron_rate_history.append(dnr_val)
+        self.epoch += 1
+
+    def to_dict(self) -> Dict[str, Any]:
+        return {
+            "epoch": int(self.epoch),
+            "qe_history": list(self.qe_history),
+            "te_history": list(self.te_history),
+            "kl_history": list(self.kl_history),
+            "var_explained_history": list(self.var_explained_history),
+            "dead_neuron_rate_history": list(self.dead_neuron_rate_history),
+        }
+
+
+# ============================================================================
+# Funções auxiliares
+# ============================================================================
+def _grid_neighbors_set(
+    idx: Tuple[int, int, int, int],
+    grid_shape: Tuple[int, int, int, int],
+) -> set:
+    """Retorna o conjunto de índices vizinhos diretos (6-connectivity em 4D)
+    de um neurônio na grade."""
+    i, j, k, l = idx
+    I, J, K, L = grid_shape
+    neighbors = set()
+    for di, dj, dk, dl in [
+        (1, 0, 0, 0), (-1, 0, 0, 0),
+        (0, 1, 0, 0), (0, -1, 0, 0),
+        (0, 0, 1, 0), (0, 0, -1, 0),
+        (0, 0, 0, 1), (0, 0, 0, -1),
+    ]:
+        ni, nj, nk, nl = i + di, j + dj, k + dk, l + dl
+        if 0 <= ni < I and 0 <= nj < J and 0 <= nk < K and 0 <= nl < L:
+            neighbors.add((ni, nj, nk, nl))
+    return neighbors
+
+
+def _find_bmu_and_second(
+    x: torch.Tensor, weights: torch.Tensor
+) -> Tuple[Tuple[int, int, int, int], Tuple[int, int, int, int], float, float]:
+    """Encontra BMU e 2ª BMU para um vetor x.
+
+    Args:
+        x: tensor [4] — vetor 4D de entrada.
+        weights: tensor [I, J, K, L, 4] — pesos do SOM.
+
+    Returns:
+        (bmu_idx, second_bmu_idx, dist_bmu, dist_second)
+    """
+    grid_shape = weights.shape[:-1]
+    flat_w = weights.view(-1, 4)  # (I*J*K*L, 4)
+    diff = flat_w - x.view(1, 4)
+    dist_sq = torch.sum(diff * diff, dim=-1)  # (I*J*K*L,)
+
+    # Top-2 menores distâncias
+    top2 = torch.topk(dist_sq, k=2, largest=False)
+    flat_idx_bmu = int(top2.indices[0].item())
+    flat_idx_2nd = int(top2.indices[1].item())
+
+    # Unravel
+    L = grid_shape[3]
+    K = grid_shape[2]
+    J = grid_shape[1]
+
+    def unravel(flat_idx: int) -> Tuple[int, int, int, int]:
+        i = flat_idx // (J * K * L)
+        r = flat_idx % (J * K * L)
+        j = r // (K * L)
+        r = r % (K * L)
+        k = r // L
+        l = r % L
+        return (i, j, k, l)
+
+    bmu = unravel(flat_idx_bmu)
+    second = unravel(flat_idx_2nd)
+    dist_bmu = float(math.sqrt(max(top2.values[0].item(), 0.0)))
+    dist_second = float(math.sqrt(max(top2.values[1].item(), 0.0)))
+    return bmu, second, dist_bmu, dist_second
+
+
+# ============================================================================
+# Métricas principais (1.1 - 1.4)
+# ============================================================================
+def quantization_error(
+    data: torch.Tensor, weights: torch.Tensor
+) -> float:
+    """1.1 Erro de Quantização (QE).
+
+    QE = (1/N) * Σ_i ||x_i - W_BMU(x_i)||₂
+
+    Args:
+        data: tensor [N, 4] — vetores de entrada.
+        weights: tensor [I, J, K, L, 4] — pesos do SOM.
+
+    Returns:
+        QE (float). 0.0 se data vazio. NaN propagado é substituído por 0.0
+        (V6.5-V2-metrics-FIX-2: user requirement "investigar QE e KL resultando
+        em NAN").
+    """
+    if data.numel() == 0:
+        return 0.0
+    flat_w = weights.view(-1, 4)  # (M, 4)
+    # distância de cada amostra a todos os neurônios
+    # data: (N, 4), flat_w: (M, 4) -> (N, M)
+    diff = data.unsqueeze(1) - flat_w.unsqueeze(0)
+    dist_sq = torch.sum(diff * diff, dim=-1)  # (N, M)
+    min_dist_sq, _ = torch.min(dist_sq, dim=1)  # (N,)
+    # V6.5-V2-metrics-FIX-2: clamp para evitar sqrt de número negativo (float err)
+    min_dist_sq = torch.clamp(min_dist_sq, min=0.0)
+    qe = torch.mean(torch.sqrt(min_dist_sq))
+    qe_val = float(qe.item())
+    # NaN check final (defense in depth)
+    if qe_val != qe_val:  # NaN
+        return 0.0
+    return qe_val
+
+
+def topological_error(
+    data: torch.Tensor, weights: torch.Tensor
+) -> float:
+    """1.2 Erro Topológico (TE).
+
+    TE = (1/N) * Σ_i 𝟙[¬adjacent(BMU_1(x_i), BMU_2(x_i))]
+
+    Dois neurônios são adjacentes se differem em exatamente uma coordenada
+    por ±1 (6-connectivity em 4D).
+
+    Args:
+        data: tensor [N, 4]
+        weights: tensor [I, J, K, L, 4]
+
+    Returns:
+        TE ∈ [0, 1]. 0.0 se data vazio.
+    """
+    if data.numel() == 0:
+        return 0.0
+    grid_shape = weights.shape[:-1]
+    flat_w = weights.view(-1, 4)
+    diff = data.unsqueeze(1) - flat_w.unsqueeze(0)
+    dist_sq = torch.sum(diff * diff, dim=-1)  # (N, M)
+
+    # Top-2 BMUs para cada amostra
+    top2 = torch.topk(dist_sq, k=2, largest=False, dim=1)
+    flat_bmu = top2.indices[:, 0]  # (N,)
+    flat_2nd = top2.indices[:, 1]  # (N,)
+
+    # Unravel
+    L = grid_shape[3]
+    K = grid_shape[2]
+    J = grid_shape[1]
+    I = grid_shape[0]
+
+    def unravel_batch(flat_idx: torch.Tensor) -> torch.Tensor:
+        """Retorna tensor (N, 4) com (i, j, k, l)."""
+        i = flat_idx // (J * K * L)
+        r = flat_idx % (J * K * L)
+        j = r // (K * L)
+        r = r % (K * L)
+        k = r // L
+        l = r % L
+        return torch.stack([i, j, k, l], dim=1).long()
+
+    bmu_idx = unravel_batch(flat_bmu)  # (N, 4)
+    second_idx = unravel_batch(flat_2nd)  # (N, 4)
+
+    # Adjacência: |Δcoord| soma = 1 (apenas uma coordenada muda por ±1)
+    delta = (bmu_idx - second_idx).abs()  # (N, 4)
+    is_adjacent = (delta.sum(dim=1) == 1) & (delta.max(dim=1).values == 1)
+
+    te = 1.0 - float(is_adjacent.float().mean().item())
+    return te
+
+
+def kaski_lagus_error(
+    data: torch.Tensor, weights: torch.Tensor, alpha: float = 0.5
+) -> float:
+    """1.3 Erro de Kaski-Lagus.
+
+    Combinação ponderada de QE normalizado e TE.
+
+    Variante implementada (Kaski & Lagus, 1999):
+        KL = α * MQE_norm + (1-α) * TE
+    onde MQE_norm = QE / mean(||x||) (normalização por escala dos dados).
+
+    Uma variante alternativa multiplica:
+        KL = QE * (1 + TE)
+    que penaliza QE quando TE é alto (usada se use_multiplicative=True).
+
+    Args:
+        data: tensor [N, 4]
+        weights: tensor [I, J, K, L, 4]
+        alpha: peso do QE normalizado (default 0.5)
+
+    Returns:
+        KL (float). 0.0 se data vazio ou se NaN for detectado
+        (V6.5-V2-metrics-FIX-2: user requirement "investigar QE e KL resultando
+        em NAN").
+    """
+    if data.numel() == 0:
+        return 0.0
+    qe = quantization_error(data, weights)
+    te = topological_error(data, weights)
+    # Normalização: escala típica dos dados
+    data_norm = float(torch.mean(torch.norm(data, dim=1)).item())
+    # V6.5-V2-metrics-FIX-2: se data_norm é NaN ou zero, usa 1.0 como fallback
+    if data_norm != data_norm or data_norm < 1e-8:  # NaN or ~0
+        data_norm = 1.0
+    qe_norm = qe / max(data_norm, 1e-8)
+    kl = alpha * qe_norm + (1.0 - alpha) * te
+    kl_val = float(kl)
+    # NaN check final
+    if kl_val != kl_val:  # NaN
+        return 0.0
+    return kl_val
+
+
+def explained_variance_share(
+    data: torch.Tensor, weights: torch.Tensor
+) -> float:
+    """1.4 Variância Explicada (Share of Explained Variance).
+
+    VarExplained = 1 - Var(residual) / Var(total)
+    onde residual = x_i - W_BMU(x_i)
+
+    Implementação: computamos a variância total dos dados (soma das variâncias
+    por dimensão) e a variância dos resíduos. A fração explicada é:
+
+        VE = 1 - Σ_d Var(residual_d) / Σ_d Var(data_d)
+
+    Returns:
+        VE ∈ [0, 1]. Próximo de 1 = boa representação.
+    """
+    if data.numel() == 0 or data.shape[0] < 2:
+        return 0.0
+    flat_w = weights.view(-1, 4)
+    diff = data.unsqueeze(1) - flat_w.unsqueeze(0)
+    dist_sq = torch.sum(diff * diff, dim=-1)  # (N, M)
+    _, bmu_flat = torch.min(dist_sq, dim=1)  # (N,)
+    bmu_w = flat_w[bmu_flat]  # (N, 4)
+    residual = data - bmu_w  # (N, 4)
+
+    # Variância por dimensão (sem viés, ddof=0)
+    var_data = torch.var(data, dim=0, unbiased=False)  # (4,)
+    var_resid = torch.var(residual, dim=0, unbiased=False)  # (4,)
+
+    total_var = float(torch.sum(var_data).item())
+    resid_var = float(torch.sum(var_resid).item())
+    if total_var < 1e-12:
+        return 0.0
+    ve = 1.0 - (resid_var / total_var)
+    return float(max(0.0, min(1.0, ve)))
+
+
+# ============================================================================
+# Indicadores de falha (2.1 - 2.4)
+# ============================================================================
+def topological_collapse(weights: torch.Tensor) -> Dict[str, Any]:
+    """2.1 Colapso Topológico (Topological Collapse).
+
+    Detecta se os pesos colapsaram para um ponto (rank < 1) ou para uma linha
+    (rank < 2). Usa SVD sobre os pesos reshaped (M, 4).
+
+    Args:
+        weights: tensor [I, J, K, L, 4]
+
+    Returns:
+        Dict com:
+          - collapsed_to_point: bool (todos iguais)
+          - collapsed_to_line: bool (variação só em 1 direção)
+          - effective_rank: float (rank efetivo via razão de singular values)
+          - singular_values: list[float]
+          - severity: str ("none" | "line" | "point")
+    """
+    flat_w = weights.view(-1, 4).clone()  # (M, 4)
+    if flat_w.numel() == 0:
+        return {"collapsed_to_point": False, "collapsed_to_line": False,
+                "effective_rank": 0.0, "singular_values": [], "severity": "none"}
+
+    # Centraliza
+    flat_w_centered = flat_w - flat_w.mean(dim=0, keepdim=True)
+
+    # V6.5-V2-metrics FIX: usa SVD com fallback robusto para matrizes
+    # singulares/ill-conditioned. Se SVD falhar, usa np.linalg.eigh com
+    # regularização (adiciona pequeno ruído diagonal).
+    S_list: List[float] = []
+    try:
+        # Tentativa 1: SVD direto (mais estável que eigh)
+        U, S, V = torch.linalg.svd(flat_w_centered, full_matrices=False)
+        S_list = [float(s.item()) for s in S]
+    except Exception:
+        # Tentativa 2: eigendecomposição da matriz de covariância com regularização
+        try:
+            cov = flat_w_centered.t() @ flat_w_centered
+            # Regularização: adiciona pequeno valor diagonal para estabilizar
+            n = cov.shape[0]
+            reg = 1e-6 * float(cov.diag().max().item()) if n > 0 else 1e-6
+            cov_reg = cov + torch.eye(n) * reg
+            S_eigh, _ = torch.linalg.eigh(cov_reg)
+            S_eigh = S_eigh.flip(0)  # descending order
+            # Converte autovalores em "singular values" aproximados (sqrt)
+            S_list = [float(math.sqrt(max(s.item(), 0.0))) for s in S_eigh]
+        except Exception:
+            # Tentativa 3: usa norma de Frobenius como aproximação
+            s_frob = float(torch.norm(flat_w_centered).item())
+            S_list = [s_frob, 0.0, 0.0, 0.0]
+
+    if not S_list:
+        return {"collapsed_to_point": False, "collapsed_to_line": False,
+                "effective_rank": 0.0, "singular_values": [], "severity": "none"}
+
+    s_max = max(S_list[0] if S_list else 0.0, 1e-12)
+
+    # Effective rank: número de singular values > 1% do maior
+    threshold = 0.01 * s_max
+    n_significant = sum(1 for s in S_list if s > threshold)
+    # Effective rank (float, via entropy)
+    s_norm = [s / s_max for s in S_list]
+    s_norm = [s for s in s_norm if s > 1e-10]
+    if s_norm:
+        p = torch.tensor(s_norm)
+        p_norm = p / p.sum()
+        entropy = -float(torch.sum(p_norm * torch.log(p_norm)).item())
+        effective_rank = math.exp(entropy)
+    else:
+        effective_rank = 0.0
+
+    collapsed_to_point = n_significant <= 1 and s_max < 1e-6
+    collapsed_to_line = n_significant == 1 and s_max >= 1e-6
+
+    if collapsed_to_point:
+        severity = "point"
+    elif collapsed_to_line:
+        severity = "line"
+    else:
+        severity = "none"
+
+    return {
+        "collapsed_to_point": bool(collapsed_to_point),
+        "collapsed_to_line": bool(collapsed_to_line),
+        "effective_rank": float(effective_rank),
+        "singular_values": S_list,
+        "severity": severity,
+    }
+
+
+def dead_neuron_rate(
+    data: torch.Tensor, weights: torch.Tensor
+) -> Dict[str, Any]:
+    """2.2 Taxa de Neurônios Mortos (Dead Neurons).
+
+    DeadRate = (n_neurons - n_active_neurons) / n_neurons
+
+    Um neurônio é "ativo" se é BMU para pelo menos uma amostra.
+
+    Args:
+        data: tensor [N, 4]
+        weights: tensor [I, J, K, L, 4]
+
+    Returns:
+        Dict com:
+          - dead_neuron_rate: float ∈ [0, 1]
+          - n_dead: int
+          - n_active: int
+          - n_total: int
+          - dead_indices: list de (i,j,k,l)
+    """
+    n_total = weights.shape[0] * weights.shape[1] * weights.shape[2] * weights.shape[3]
+    if data.numel() == 0:
+        return {"dead_neuron_rate": 1.0, "n_dead": n_total, "n_active": 0,
+                "n_total": n_total, "dead_indices": []}
+
+    flat_w = weights.view(-1, 4)
+    diff = data.unsqueeze(1) - flat_w.unsqueeze(0)
+    dist_sq = torch.sum(diff * diff, dim=-1)  # (N, M)
+    _, bmu_flat = torch.min(dist_sq, dim=1)  # (N,)
+
+    active_mask = torch.zeros(n_total, dtype=torch.bool)
+    active_mask[bmu_flat] = True
+    n_active = int(active_mask.sum().item())
+    n_dead = n_total - n_active
+
+    # Coleta índices mortos
+    dead_flat = (~active_mask).nonzero(as_tuple=True)[0]
+    L_dim = weights.shape[3]
+    K_dim = weights.shape[2]
+    J_dim = weights.shape[1]
+    dead_indices = []
+    for flat_idx in dead_flat.tolist():
+        i = flat_idx // (J_dim * K_dim * L_dim)
+        r = flat_idx % (J_dim * K_dim * L_dim)
+        j = r // (K_dim * L_dim)
+        r = r % (K_dim * L_dim)
+        k = r // L_dim
+        l = r % L_dim
+        dead_indices.append((i, j, k, l))
+
+    return {
+        "dead_neuron_rate": float(n_dead / n_total) if n_total > 0 else 0.0,
+        "n_dead": int(n_dead),
+        "n_active": int(n_active),
+        "n_total": int(n_total),
+        "dead_indices": dead_indices[:50],  # limita para não explodir JSON
+    }
+
+
+def quantization_error_stagnation(
+    history: SOMMetricHistory,
+    high_threshold_frac: float = 0.7,
+    slope_tolerance: float = 1e-4,
+    window_size: int = 5,
+) -> Dict[str, Any]:
+    """2.3 Estagnação do Erro de Quantização.
+
+    Detecta se o QE estabilizou em patamar elevado:
+      - |slope(QE)| < slope_tolerance (estável)
+      - QE_atual > high_threshold_frac * max(QE_history) (alto)
+
+    Args:
+        history: histórico de métricas SOM.
+        high_threshold_frac: fração do QE máximo para considerar "alto".
+        slope_tolerance: tolerância de slope para considerar "estável".
+        window_size: nº de épocas recentes para regressão linear.
+
+    Returns:
+        Dict com:
+          - is_stagnant: bool
+          - current_qe: float
+          - max_qe: float
+          - slope: float
+          - severity: str ("none" | "mild" | "severe")
+    """
+    qe_list = list(history.qe_history)
+    if len(qe_list) < window_size + 2:
+        return {"is_stagnant": False, "current_qe": 0.0, "max_qe": 0.0,
+                "slope": 0.0, "severity": "none", "n_epochs": len(qe_list)}
+
+    # Regressão linear sobre as últimas window_size épocas
+    recent = qe_list[-window_size:]
+    n = len(recent)
+    x_mean = (n - 1) / 2.0
+    y_mean = sum(recent) / n
+    num = sum((i - x_mean) * (y - y_mean) for i, y in enumerate(recent))
+    den = sum((i - x_mean) ** 2 for i in range(n))
+    slope = num / den if den > 0 else 0.0
+
+    current_qe = float(qe_list[-1])
+    max_qe = float(max(qe_list))
+    is_high = current_qe > high_threshold_frac * max_qe
+    is_stable = abs(slope) < slope_tolerance
+
+    is_stagnant = is_stable and is_high
+
+    if is_stagnant:
+        if current_qe > 0.9 * max_qe:
+            severity = "severe"
+        else:
+            severity = "mild"
+    else:
+        severity = "none"
+
+    return {
+        "is_stagnant": bool(is_stagnant),
+        "current_qe": current_qe,
+        "max_qe": max_qe,
+        "slope": float(slope),
+        "severity": severity,
+        "n_epochs": len(qe_list),
+    }
+
+
+def neighborhood_crossing(
+    history: SOMMetricHistory,
+    initial_window: int = 5,
+    final_window: int = 5,
+    growth_threshold: float = 1.5,
+) -> Dict[str, Any]:
+    """2.4 Cruzamento de Vizinhança (Neighborhood Crossing).
+
+    Detecta crescimento repentino ou oscilação crônica do TE nas épocas
+    finais:
+      - std(TE_final) > growth_threshold * std(TE_initial) (oscilação cresceu)
+      - TE[-1] > TE[-5] (crescimento no final)
+
+    Args:
+        history: histórico de métricas SOM.
+        initial_window: nº de épocas iniciais para baseline.
+        final_window: nº de épocas finais para comparação.
+        growth_threshold: razão de std para considerar "crescimento crônico".
+
+    Returns:
+        Dict com:
+          - detected: bool
+          - te_initial_mean: float
+          - te_final_mean: float
+          - te_initial_std: float
+          - te_final_std: float
+          - std_ratio: float
+          - is_oscillating: bool
+          - is_growing: bool
+          - severity: str ("none" | "mild" | "severe")
+    """
+    te_list = list(history.te_history)
+    if len(te_list) < initial_window + final_window:
+        return {"detected": False, "te_initial_mean": 0.0, "te_final_mean": 0.0,
+                "te_initial_std": 0.0, "te_final_std": 0.0, "std_ratio": 0.0,
+                "is_oscillating": False, "is_growing": False, "severity": "none",
+                "n_epochs": len(te_list)}
+
+    initial = te_list[:initial_window]
+    final = te_list[-final_window:]
+
+    init_mean = sum(initial) / len(initial)
+    final_mean = sum(final) / len(final)
+    init_var = sum((x - init_mean) ** 2 for x in initial) / len(initial)
+    final_var = sum((x - final_mean) ** 2 for x in final) / len(final)
+    init_std = math.sqrt(init_var)
+    final_std = math.sqrt(final_var)
+
+    std_ratio = final_std / max(init_std, 1e-8)
+    is_oscillating = std_ratio > growth_threshold
+    is_growing = final_mean > init_mean and (final_mean - init_mean) > 0.05
+
+    detected = is_oscillating or is_growing
+
+    if detected:
+        if std_ratio > 2.0 * growth_threshold or is_growing:
+            severity = "severe"
+        else:
+            severity = "mild"
+    else:
+        severity = "none"
+
+    return {
+        "detected": bool(detected),
+        "te_initial_mean": float(init_mean),
+        "te_final_mean": float(final_mean),
+        "te_initial_std": float(init_std),
+        "te_final_std": float(final_std),
+        "std_ratio": float(std_ratio),
+        "is_oscillating": bool(is_oscillating),
+        "is_growing": bool(is_growing),
+        "severity": severity,
+        "n_epochs": len(te_list),
+    }
+
+
+# ============================================================================
+# Função agregadora: compute_all_metrics
+# ============================================================================
+def compute_all_metrics(
+    data: torch.Tensor,
+    weights: torch.Tensor,
+    history: Optional[SOMMetricHistory] = None,
+) -> Dict[str, Any]:
+    """Computa todas as métricas SOM de uma vez.
+
+    Args:
+        data: tensor [N, 4] — buffer de vetores 4D.
+        weights: tensor [I, J, K, L, 4] — pesos do SOM.
+        history: histórico de métricas (opcional, para estagnação e cruzamento).
+
+    Returns:
+        Dict com:
+          - quantization_error: float
+          - topological_error: float
+          - kaski_lagus_error: float
+          - explained_variance_share: float
+          - topological_collapse: Dict
+          - dead_neuron_rate: Dict
+          - quantization_error_stagnation: Dict (se history fornecido)
+          - neighborhood_crossing: Dict (se history fornecido)
+          - overall_health: str ("healthy" | "warning" | "critical")
+          - failure_indicators: list[str]
+    """
+    # V6.5-V2-metrics FIX: wrap entire computation in try/except para evitar
+    # crashes fatais quando o SOM tem pesos degenerados (ex: todos iguais
+    # após muito treinamento sem diversidade). Em caso de erro, retorna
+    # métricas safe-default em vez de crashar o treino.
+
+    # V6.5-V2-metrics-FIX-2 (user requirement: "investigar QE e KL resultando
+    # em NAN"): detecção EXPLÍCITA de NaN/Inf nos tensores de entrada ANTES
+    # de computar as métricas. Se os pesos do SOM contêm NaN (devido a
+    # gradientes explosivos durante train_hypotheses ou apply_best_delta),
+    # TODAS as métricas (QE, KL, etc.) seriam NaN, propagando o erro para
+    # o histórico e quebrando a detecção de estagnação/cruzamento.
+    # Esta barreira inicial retorna safe-defaults imediatamente quando
+    # NaN/Inf é detectado, evitando contaminação do histórico.
+    try:
+        if data is not None and data.numel() > 0:
+            if bool(torch.isnan(data).any().item()) or bool(torch.isinf(data).any().item()):
+                n_nan = int(torch.isnan(data).sum().item())
+                n_inf = int(torch.isinf(data).sum().item())
+                logger.warning(
+                    f"[som_metrics] NaN/Inf detected in INPUT DATA: "
+                    f"{n_nan} NaN, {n_inf} Inf out of {data.numel()} elements. "
+                    f"Returning safe-default metrics (skipping computation)."
+                )
+                return _nan_safe_metrics(
+                    reason=f"nan_in_data (nan={n_nan}, inf={n_inf})",
+                    data_shape=tuple(data.shape),
+                    weights_shape=tuple(weights.shape) if weights is not None else None,
+                )
+        if weights is not None and weights.numel() > 0:
+            if bool(torch.isnan(weights).any().item()) or bool(torch.isinf(weights).any().item()):
+                n_nan = int(torch.isnan(weights).sum().item())
+                n_inf = int(torch.isinf(weights).sum().item())
+                logger.warning(
+                    f"[som_metrics] NaN/Inf detected in SOM WEIGHTS: "
+                    f"{n_nan} NaN, {n_inf} Inf out of {weights.numel()} elements. "
+                    f"Returning safe-default metrics (skipping computation). "
+                    f"This usually indicates exploding gradients in "
+                    f"train_hypotheses() or apply_best_delta_and_consolidate()."
+                )
+                return _nan_safe_metrics(
+                    reason=f"nan_in_weights (nan={n_nan}, inf={n_inf})",
+                    data_shape=tuple(data.shape) if data is not None else None,
+                    weights_shape=tuple(weights.shape),
+                )
+    except Exception as nan_check_err:
+        logger.warning(
+            f"[som_metrics] NaN pre-check failed (proceeding to compute): {nan_check_err}"
+        )
+
+    try:
+        result = _compute_all_metrics_impl(data, weights, history)
+        # V6.5-V2-metrics-FIX-2: pós-computação, verifica se QE ou KL é NaN.
+        # Se sim, registra warning e substitui por 0.0 (para não quebrar logs JSON
+        # e nem contaminar o histórico de estagnação/cruzamento).
+        qe_val = result.get("quantization_error", 0.0)
+        kl_val = result.get("kaski_lagus_error", 0.0)
+        if isinstance(qe_val, float) and (qe_val != qe_val):  # NaN check
+            logger.warning(
+                f"[som_metrics] QE computed as NaN — replacing with 0.0 "
+                f"(indicates degenerate SOM state). Investigate train_hypotheses "
+                f"or apply_best_delta for gradient explosion."
+            )
+            result["quantization_error"] = 0.0
+            result["qe_was_nan"] = True
+            # KL depende de QE, então também será NaN
+            result["kaski_lagus_error"] = 0.0
+            result["kl_was_nan"] = True
+        elif isinstance(kl_val, float) and (kl_val != kl_val):  # NaN check
+            logger.warning(
+                f"[som_metrics] KL computed as NaN (QE was fine) — replacing with 0.0. "
+                f"Usually due to zero-norm data in kaski_lagus normalization."
+            )
+            result["kaski_lagus_error"] = 0.0
+            result["kl_was_nan"] = True
+        return result
+    except Exception as e:
+        logger.warning(
+            f"[som_metrics] compute_all_metrics failed (using safe defaults): {e}"
+        )
+        return _nan_safe_metrics(reason=f"compute_error: {str(e)[:200]}", data_shape=None, weights_shape=None)
+
+
+def _nan_safe_metrics(
+    reason: str,
+    data_shape: Optional[Tuple[int, ...]] = None,
+    weights_shape: Optional[Tuple[int, ...]] = None,
+) -> Dict[str, Any]:
+    """V6.5-V2-metrics-FIX-2 — Retorna métricas safe-default quando NaN é detectado.
+
+    User requirement: "Aprimorar a FASE2 PUNITIVA investigar QE e KL resultando em NAN".
+
+    Este helper centraliza a resposta a NaN/Inf em qualquer ponto do pipeline
+    de métricas, garantindo que:
+      1. O treino NÃO crashe.
+      2. O histórico de métricas (SOMMetricHistory) NÃO seja contaminado com NaN
+         (o que quebraria detecção de estagnação e cruzamento de vizinhança).
+      3. O log indique claramente a fonte do NaN (data vs weights vs compute_error).
+      4. failure_indicators registre o evento para análise pós-treino.
+    """
+    return {
+        "quantization_error": 0.0,
+        "topological_error": 0.0,
+        "kaski_lagus_error": 0.0,
+        "explained_variance_share": 0.0,
+        "topological_collapse": {
+            "collapsed_to_point": False, "collapsed_to_line": False,
+            "effective_rank": 0.0, "singular_values": [],
+            "severity": "none", "error": reason,
+        },
+        "dead_neuron_rate": {
+            "dead_neuron_rate": 1.0, "n_dead": 0, "n_active": 0,
+            "n_total": 0, "dead_indices": [], "error": reason,
+        },
+        "qe_stagnation": {
+            "is_stagnant": False, "current_qe": 0.0, "max_qe": 0.0,
+            "slope": 0.0, "severity": "none", "n_epochs": 0,
+        },
+        "neighborhood_crossing": {
+            "detected": False, "te_initial_mean": 0.0, "te_final_mean": 0.0,
+            "te_initial_std": 0.0, "te_final_std": 0.0, "std_ratio": 0.0,
+            "is_oscillating": False, "is_growing": False, "severity": "none",
+            "n_epochs": 0,
+        },
+        "overall_health": "critical",
+        "failure_indicators": [f"nan_detected: {reason[:120]}"],
+        "n_failure_indicators": 1,
+        "nan_detected": True,
+        "nan_reason": reason,
+        "data_shape": list(data_shape) if data_shape else None,
+        "weights_shape": list(weights_shape) if weights_shape else None,
+    }
+
+
+def _compute_all_metrics_impl(
+    data: torch.Tensor,
+    weights: torch.Tensor,
+    history: Optional[SOMMetricHistory] = None,
+) -> Dict[str, Any]:
+    """Implementação interna de compute_all_metrics (sem try/except wrapper)."""
+    metrics: Dict[str, Any] = {}
+
+    # 1.1 - 1.4: Métricas principais
+    metrics["quantization_error"] = quantization_error(data, weights)
+    metrics["topological_error"] = topological_error(data, weights)
+    metrics["kaski_lagus_error"] = kaski_lagus_error(data, weights)
+    metrics["explained_variance_share"] = explained_variance_share(data, weights)
+
+    # 2.1: Colapso topológico
+    collapse = topological_collapse(weights)
+    metrics["topological_collapse"] = collapse
+
+    # 2.2: Dead neurons
+    dead = dead_neuron_rate(data, weights)
+    metrics["dead_neuron_rate"] = dead
+
+    # 2.3: Estagnação do QE (requer histórico)
+    if history is not None and len(history.qe_history) > 5:
+        metrics["qe_stagnation"] = quantization_error_stagnation(history)
+    else:
+        metrics["qe_stagnation"] = {
+            "is_stagnant": False, "current_qe": metrics["quantization_error"],
+            "max_qe": metrics["quantization_error"], "slope": 0.0,
+            "severity": "none", "n_epochs": 0,
+        }
+
+    # 2.4: Cruzamento de vizinhança (requer histórico)
+    if history is not None and len(history.te_history) > 10:
+        metrics["neighborhood_crossing"] = neighborhood_crossing(history)
+    else:
+        metrics["neighborhood_crossing"] = {
+            "detected": False, "te_initial_mean": 0.0, "te_final_mean": 0.0,
+            "te_initial_std": 0.0, "te_final_std": 0.0, "std_ratio": 0.0,
+            "is_oscillating": False, "is_growing": False, "severity": "none",
+            "n_epochs": 0,
+        }
+
+    # Avaliação de saúde geral
+    failure_indicators: List[str] = []
+    if collapse["collapsed_to_point"]:
+        failure_indicators.append("topological_collapse_point")
+    elif collapse["collapsed_to_line"]:
+        failure_indicators.append("topological_collapse_line")
+    if dead["dead_neuron_rate"] > 0.5:
+        failure_indicators.append(f"dead_neuron_rate_high_{dead['dead_neuron_rate']:.2f}")
+    if metrics["qe_stagnation"]["is_stagnant"]:
+        failure_indicators.append(f"qe_stagnant_{metrics['qe_stagnation']['severity']}")
+    if metrics["neighborhood_crossing"]["detected"]:
+        failure_indicators.append(
+            f"neighborhood_crossing_{metrics['neighborhood_crossing']['severity']}"
+        )
+    if metrics["explained_variance_share"] < 0.3:
+        failure_indicators.append(
+            f"low_explained_variance_{metrics['explained_variance_share']:.2f}"
+        )
+    if metrics["topological_error"] > 0.5:
+        failure_indicators.append(
+            f"high_topological_error_{metrics['topological_error']:.2f}"
+        )
+
+    if len(failure_indicators) >= 2:
+        overall_health = "critical"
+    elif len(failure_indicators) == 1:
+        overall_health = "warning"
+    else:
+        overall_health = "healthy"
+
+    metrics["failure_indicators"] = failure_indicators
+    metrics["overall_health"] = overall_health
+    metrics["n_failure_indicators"] = len(failure_indicators)
+
+    return metrics
+
+
+__all__ = [
+    "SOMMetricHistory",
+    "quantization_error",
+    "topological_error",
+    "kaski_lagus_error",
+    "explained_variance_share",
+    "topological_collapse",
+    "dead_neuron_rate",
+    "quantization_error_stagnation",
+    "neighborhood_crossing",
+    "compute_all_metrics",
+]

streaming_datasets.py

@@ -0,0 +1,995 @@
+"""streaming_datasets_v13_9.py — Streaming dataset loader for v13.9.1 training.
+
+Carrega os 9 datasets especificados pelo usuário via streaming (IterableDataset),
+um por vez, exaustivamente dentro do limite max_samples_per_dataset.
+
+Datasets:
+  1. CEIA-POSITIVO/ultrachat_br_clustred_balanced_v1
+  2. Madras1/corpus-ptbr-v2
+  3. rhaymison/multmodal_175k_portuguese
+  4. TucanoBR/GigaVerbo
+  5. nvidia/OpenMathReasoning
+  6. MathLLMs/MathVision
+  7. nvidia/OpenMathInstruct-2
+  8. dominguesm/restore-punctuation-ptbr-dataset
+  9. carolina-c4ai/corpus-carolina
+
+Memory-efficient: streaming=True, no full materialization.
+Returns ProcessedSample with raw_text only (tokenizer handles encoding).
+"""
+from __future__ import annotations
+
+import logging
+import os
+import time
+import traceback
+from dataclasses import dataclass, field
+from typing import Any, Dict, Iterator, List, Optional, Tuple
+
+import torch
+
+logger = logging.getLogger(__name__)
+
+# Datasets V13.9.1 — exatamente os 9 especificados pelo usuário
+# V6.5-final: adicionado dominguesm/Canarim-Instruct-PTBR-Dataset
+# V6.5-7ds: adicionados adalbertojunior/punctuation-ptbr,
+#           iara-project/news-articles-ptbr-dataset, manoela/noticias_ptbr
+# V6.5-V2: adicionado BrunoN-Dev/corpus-ptbr-v1 como 8º dataset de CONHECIMENTO
+#          e usado como dataset de TREINAMENTO COM PUNIÇÃO
+DEFAULT_DATASETS = [
+    "CEIA-POSITIVO/ultrachat_br_clustred_balanced_v1",
+    "Madras1/corpus-ptbr-v2",
+    "rhaymison/multmodal_175k_portuguese",
+    "TucanoBR/GigaVerbo",
+    "nvidia/OpenMathReasoning",
+    "MathLLMs/MathVision",
+    "nvidia/OpenMathInstruct-2",
+    "dominguesm/restore-punctuation-ptbr-dataset",
+    "carolina-c4ai/corpus-carolina",
+    "dominguesm/Canarim-Instruct-PTBR-Dataset",
+    "adalbertojunior/punctuation-ptbr",
+    "iara-project/news-articles-ptbr-dataset",
+    "manoela/noticias_ptbr",
+    "BrunoN-Dev/corpus-ptbr-v1",
+]
+
+# V6.5-V2 — Sequência de 8 datasets para a fase CONHECIMENTO (na ordem exata
+# especificada pelo usuário). Meta mínima: 8000 samples ou mais.
+CONHECIMENTO_DATASETS_V2 = [
+    "dominguesm/restore-punctuation-ptbr-dataset",
+    "carolina-c4ai/corpus-carolina",
+    "CEIA-POSITIVO/ultrachat_br_clustred_balanced_v1",
+    "dominguesm/Canarim-Instruct-PTBR-Dataset",
+    "adalbertojunior/punctuation-ptbr",
+    "iara-project/news-articles-ptbr-dataset",
+    "manoela/noticias_ptbr",
+    "BrunoN-Dev/corpus-ptbr-v1",
+]
+
+# V6.5-V2 — Dataset para TREINAMENTO COM PUNIÇÃO (500 samples).
+# O usuário especificou: "dataset para TREINAMENTO COM PUNIÇÃO ATIVA
+# 'BrunoN-Dev/corpus-ptbr-v1'".
+PUNICAO_DATASET_V2 = "BrunoN-Dev/corpus-ptbr-v1"
+
+# Dataset format descriptors — Lista priorizada de campos de texto
+DATASET_FORMATS: Dict[str, Dict[str, Any]] = {
+    "CEIA-POSITIVO/ultrachat_br_clustred_balanced_v1": {
+        "type": "chat",
+        "text_fields": ["conversa", "text", "content", "conversation", "messages"],
+        "label_fields": [],
+        "split": "train",
+        "config": None,
+    },
+    "Madras1/corpus-ptbr-v2": {
+        "type": "text",
+        "text_fields": ["text", "content", "document", "body"],
+        "label_fields": [],
+        "split": "train",
+        "config": None,
+    },
+    "rhaymison/multmodal_175k_portuguese": {
+        "type": "multimodal",
+        "text_fields": ["description", "text", "question", "prompt", "instruction", "input"],
+        "image_fields": ["image", "image_url"],
+        "label_fields": ["answer", "response", "output"],
+        "split": "train",
+        "config": None,
+    },
+    "TucanoBR/GigaVerbo": {
+        "type": "text",
+        "text_fields": ["text", "content", "document", "body"],
+        "label_fields": [],
+        "split": "train",
+        "config": None,
+    },
+    "nvidia/OpenMathReasoning": {
+        "type": "math",
+        "text_fields": ["problem", "question", "input", "prompt"],
+        "label_fields": ["solution", "answer", "output", "response"],
+        # BUG FIX: OpenMathReasoning has splits ['cot', 'tir', 'genselect', 'additional_problems']
+        "split": "cot",  # Use 'cot' (chain-of-thought) as primary split
+        "config": None,
+    },
+    "MathLLMs/MathVision": {
+        "type": "math_multimodal",
+        "text_fields": ["question", "problem", "text", "query"],
+        "image_fields": ["image", "image_url"],
+        "label_fields": ["answer", "solution", "response"],
+        # BUG FIX: MathVision has only 'test' and 'testmini' splits (no train)
+        "split": "test",
+        "config": None,
+    },
+    "nvidia/OpenMathInstruct-2": {
+        "type": "math",
+        "text_fields": ["problem", "question", "input", "prompt"],
+        # V3 FIX: campos reais do dataset são generated_solution e expected_answer
+        "label_fields": ["generated_solution", "expected_answer", "solution", "answer", "output", "response"],
+        "split": "train",
+        "config": None,
+    },
+    "dominguesm/restore-punctuation-ptbr-dataset": {
+        "type": "punctuation",
+        "text_fields": ["text", "original", "unpunctuated", "input"],
+        "label_fields": ["punctuated", "restored", "target", "output"],
+        "split": "train",
+        "config": None,
+    },
+    "carolina-c4ai/corpus-carolina": {
+        "type": "text",
+        "text_fields": ["text", "content", "document", "body", "xml"],
+        "label_fields": [],
+        "split": "train",
+        "config": None,
+        # V13.9.2-carolina: dataset original usa script Python (não suportado em
+        # datasets 5.0+). Workaround: stream_xml_gz direto via lxml.iterparse,
+        # sem usar o script Python. Implementado em _stream_carolina_direct().
+    },
+    # ── V13.9.2-finetune-v2: novos datasets PT-BR ───────────────────────────
+    # Adicionados para fine-tuning contínuo do modelo v13.9.2 (preserva
+    # arquitetura e parâmetros). Nenhuma mudança estrutural — apenas novos
+    # descritores de formato para que stream_dataset() reconheça os datasets.
+    #
+    # V13.9.2-finetune-v3: padronização para o formato unificado
+    # "### Instruction:\n...\n\n### Response:\n..." conforme requisitado pelo
+    # usuário. Apenas os datasets com estrutura instruction/response (orion,
+    # cnmoro) usam o template; strak2005/bratao continuam como texto puro
+    # (corpus sem estrutura de instrução). Nenhuma mudança nos 9 datasets
+    # base — retrocompatibilidade total.
+    "orion-research/translations-en_US-pt_BR": {
+        "type": "translation",
+        # string = EN (instruction), string_translation = PT-BR (response).
+        # V13.9.2-finetune-v3: separa instruction/response para o template
+        # unificado em vez de concatenar como texto único.
+        "text_fields": ["string"],
+        "label_fields": ["string_translation"],
+        "split": "train",
+        "config": None,
+        "format_template": "instruction_response",
+        "instruction_prefix": "Translate the following text to Portuguese:",
+    },
+    "cnmoro/Instruct-PTBR-10M": {
+        "type": "instruct",
+        # NB: campos em MAIÚSCULAS no schema do dataset
+        # V13.9.2-finetune-v3: INSTRUCTION -> instruction, RESPONSE -> response
+        # via template unificado.
+        "text_fields": ["INSTRUCTION"],
+        "label_fields": ["RESPONSE"],
+        "split": "train",
+        "config": None,
+        "format_template": "instruction_response",
+        # V13.9.2-finetune-v2: dataset é um único parquet de 10 GB — streaming
+        # via datasets.load_dataset() é extremamente lento para materializar
+        # a primeira amostra. Fallback via HF datasets-server rows API (HTTP),
+        # que retorna batches de 100 rows rapidamente. Implementado em
+        # _stream_via_rows_api().
+        "loader": "rows_api",
+    },
+    "strak2005/corpus-ptbr-v1": {
+        "type": "text",
+        "text_fields": ["text", "content"],
+        "label_fields": [],
+        "split": "train",
+        "config": None,
+        # Fallback: se strak2005 cair, tenta bratao (mirror idêntico)
+        "fallback": "bratao/corpus-ptbr-v1",
+        # V13.9.2-finetune-v2: dataset tem 18 shards parquet de ~1GB cada.
+        # Streaming via datasets.load_dataset() não materializa primeira
+        # amostra em tempo aceitável. Fallback: baixa 1 shard localmente e
+        # itera com pyarrow. Implementado em _stream_via_parquet_first_shard().
+        "loader": "parquet_first_shard",
+    },
+    "bratao/corpus-ptbr-v1": {
+        "type": "text",
+        "text_fields": ["text", "content"],
+        "label_fields": [],
+        "split": "train",
+        "config": None,
+        "loader": "parquet_first_shard",
+    },
+    # ── V6.5: novos datasets para esgotar ────────────────────────────────
+    # User requirement: "esgotar 'dominguesm/restore-punctuation-ptbr-dataset'
+    # e 'carolina-c4ai/corpus-carolina' e 'nvidia/OpenMathInstruct-2' e
+    # 'nvidia/OpenMathReasoning' e 'CEIA-POSITIVO/ultrachat_br_clustred_balanced_v1'
+    # e 'Dexavator/English-PTBR'"
+    #
+    # Dexavator/English-PTBR: dataset de tradução EN->PT-BR.
+    # Estrutura típica: {"english": "...", "portuguese": "..."} ou
+    # {"en": "...", "pt": "..."}.
+    "Dexavator/English-PTBR": {
+        "type": "translation",
+        "text_fields": ["english", "en", "text", "source"],
+        "label_fields": ["portuguese", "pt", "target", "translation"],
+        "split": "train",
+        "config": None,
+        "format_template": "instruction_response",
+        "instruction_prefix": "Translate the following text to Portuguese:",
+    },
+    # ── V6.5-final: Canarim-Instruct-PTBR-Dataset ────────────────────────
+    # User requirement: "esgotar 'dominguesm/Canarim-Instruct-PTBR-Dataset'"
+    # Canarim-Instruct: dataset PT-BR de instruções (~430k samples).
+    # Estrutura típica: {"instruction": "...", "input": "...", "output": "..."}
+    # ou {"text": "...", "conversation": [{"role": "user", "content": "..."}, ...]}
+    "dominguesm/Canarim-Instruct-PTBR-Dataset": {
+        "type": "instruct",
+        "text_fields": ["instruction", "input", "text", "prompt", "question"],
+        "label_fields": ["output", "response", "answer"],
+        "split": "train",
+        "config": None,
+        "format_template": "instruction_response",
+        "instruction_prefix": "Instrução:",
+    },
+    # ── V6.5-7ds: novos 3 datasets PT-BR ─────────────────────────────────
+    # User requirement (latest): "streaming até esgotar nesta sequência
+    # 'dominguesm/restore-punctuation-ptbr-dataset' e
+    # 'carolina-c4ai/corpus-carolina' e
+    # 'CEIA-POSITIVO/ultrachat_br_clustred_balanced_v1' e
+    # 'dominguesm/Canarim-Instruct-PTBR-Dataset' e
+    # 'adalbertojunior/punctuation-ptbr' e
+    # 'iara-project/news-articles-ptbr-dataset' e
+    # 'manoela/noticias_ptbr'"
+    #
+    # adalbertojunior/punctuation-ptbr:
+    #   Dataset de restauração de pontuação PT-BR. Schema:
+    #     {id: str, tokens: list[str], pos_tags, chunk_tags, ner_tags}
+    #   O campo "tokens" é uma lista de palavras (sem pontuação). Precisamos
+    #   fazer join com espaços para obter texto bruto. Como o dataset usa
+    #   script Python (datasets 5.0+ não suporta), usamos rows_api loader.
+    #   Splits: train/validation/test com config "punctuation-ptbr".
+    "adalbertojunior/punctuation-ptbr": {
+        "type": "punctuation_tokens",
+        "text_fields": ["tokens"],  # list[str] — join com espaços
+        "label_fields": [],
+        "split": "train",
+        "config": "punctuation-ptbr",  # necessário para rows_api
+        "loader": "rows_api",
+        "join_tokens": True,  # flag especial para _normalize_sample
+    },
+    # iara-project/news-articles-ptbr-dataset:
+    #   Dataset de notícias PT-BR da Folha de S.Paulo (via IARA project).
+    #   Schema: {title, text, date, category, category_natural_language, link}
+    #   Streaming normal via parquet — primeira amostra materializa rápido.
+    #   "text" é o corpo da notícia, "title" é o título. Concatenamos ambos
+    #   via format_template="title_text_concat" para enriquecer contexto.
+    #   O campo "category" é usado como label (categoria da notícia).
+    "iara-project/news-articles-ptbr-dataset": {
+        "type": "news",
+        # text_fields em ordem de prioridade: "text" (corpo) primeiro
+        "text_fields": ["text", "title", "content", "body"],
+        "label_fields": ["category", "category_natural_language", "title"],
+        "split": "train",
+        "config": None,
+        # V6.5-7ds: title_text_concat usa title como label (prefixo)
+        # e text como corpo. Aqui label_fields pega "category" primeiro
+        # para dar contexto temático.
+        "format_template": "title_text_concat",
+    },
+    # manoela/noticias_ptbr:
+    #   Mirror do iara-project/news-articles-ptbr-dataset (mesmo parquet,
+    #   mesmo schema). Streaming normal via parquet.
+    "manoela/noticias_ptbr": {
+        "type": "news",
+        "text_fields": ["text", "title", "content", "body"],
+        "label_fields": ["category", "category_natural_language", "title"],
+        "split": "train",
+        "config": None,
+        "format_template": "title_text_concat",
+    },
+    # ── V6.5-V2: BrunoN-Dev/corpus-ptbr-v1 ───────────────────────────────
+    # User requirement: "dataset para TREINAMENTO COM PUNIÇÃO ATIVA
+    # 'BrunoN-Dev/corpus-ptbr-v1'" e o 8º dataset de CONHECIMENTO.
+    #
+    # BrunoN-Dev/corpus-ptbr-v1: corpus PT-BR genérico com 18 shards parquet
+    # (~1GB cada). Schema: {"text": "...", "meta": "..."}.
+    # Streaming via datasets.load_dataset(streaming=True) é lento para
+    # materializar a primeira amostra — usamos o loader parquet_first_shard
+    # (baixa apenas o primeiro shard e itera localmente com pyarrow).
+    # Sem label explícita — usamos hash do texto (paridade do nº de palavras)
+    # para gerar label binário (0/1) — não é dado sintético, é derivação
+    # algorítmica do conteúdo real do dataset.
+    "BrunoN-Dev/corpus-ptbr-v1": {
+        "type": "text",
+        "text_fields": ["text", "content", "body", "document"],
+        "label_fields": [],
+        "split": "train",
+        "config": None,
+        "loader": "parquet_first_shard",
+        # V6.5-V2: gera label binário determinístico via hash do texto
+        # (paridade do número de palavras) — não é dado sintético, é
+        # derivação algorítmica do conteúdo real do dataset.
+        "label_strategy": "hash_parity",
+    },
+}
+
+
+# Lista de arquivos XML.gz do Carolina que tentaremos baixar (ordem: menores
+# taxonomias primeiro para minimizar banda e RAM). Selecionamos 2 arquivos
+# por taxonomia para garantir diversidade sem explodir a memória.
+CAROLINA_XML_FILES = [
+    # taxonomia "datasets_and_other_corpora" (DAT = pt-BR notícias)
+    "corpus/datasets_and_other_corpora/pt-BR/DATa.xml.gz",
+    "corpus/datasets_and_other_corpora/pt-BR/DATb.xml.gz",
+    # taxonomia "wik" (wikis)
+    "corpus/wik/WIKa.xml.gz",
+    "corpus/wik/WIKb.xml.gz",
+    # taxonomia "uni" (university_domains)
+    "corpus/uni/UNIa.xml.gz",
+    "corpus/uni/UNIb.xml.gz",
+    # taxonomia "pub" (public_domain_works)
+    "corpus/pub/PUBa.xml.gz",
+    "corpus/pub/PUBb.xml.gz",
+]
+
+
+def _stream_carolina_direct(
+    max_samples: int,
+    hf_token: Optional[str] = None,
+    timeout_per_file: int = 60,
+) -> Iterator[Dict[str, Any]]:
+    """Workaround V13.9.2-carolina: carrega Carolina sem usar o script Python.
+
+    Baixa arquivos XML.gz diretamente do repo e faz parse streaming com
+    lxml.etree.iterparse (compatível com datasets 5.0+). Cada <TEI> vira um
+    documento de texto (concatenação dos <p> dentro de <body>).
+
+    Args:
+        max_samples: número máximo de amostras a retornar
+        hf_token: token HF opcional (dataset é público, mas token evita rate limit)
+        timeout_per_file: tempo máximo (segundos) por arquivo XML
+
+    Yields:
+        dict com chave "text" (compatível com _normalize_sample)
+    """
+    from huggingface_hub import hf_hub_download
+    import gzip
+    from lxml import etree
+
+    TEI_NS = "{http://www.tei-c.org/ns/1.0}"
+    BODY_P_TAG = f".//{TEI_NS}body/{TEI_NS}p"
+
+    count = 0
+    for xml_path in CAROLINA_XML_FILES:
+        if count >= max_samples:
+            return
+        try:
+            local_path = hf_hub_download(
+                repo_id="carolina-c4ai/corpus-carolina",
+                filename=xml_path,
+                repo_type="dataset",
+                token=hf_token,
+            )
+        except Exception as e:
+            logger.warning(f"Carolina: não foi possível baixar {xml_path}: {e}")
+            continue
+
+        try:
+            with gzip.open(local_path, "rb") as gz:
+                # iterparse streaming: não carrega árvore inteira na memória
+                for _, tei in etree.iterparse(
+                    gz, huge_tree=True, encoding="utf-8", tag=f"{TEI_NS}TEI"
+                ):
+                    if count >= max_samples:
+                        tei.clear()
+                        return
+                    # Extrai texto dos <p> dentro de <body>
+                    parts = []
+                    for p in tei.findall(BODY_P_TAG):
+                        if p.text:
+                            parts.append(p.text)
+                    text = " ".join(parts).strip()
+                    if text and len(text) >= 10:
+                        yield {"text": text, "meta": ""}
+                    tei.clear()  # libera memória da árvore TEI
+                    count += 1
+        except Exception as e:
+            logger.warning(f"Carolina: erro processando {xml_path}: {e}")
+            continue
+
+    logger.info(f"Carolina: streamou {count} amostras no total")
+
+
+# ═══════════════════════════════════════════════════════════════════════════
+# V13.9.2-finetune-v2: Loaders alternativos para datasets parquet grandes
+# onde datasets.load_dataset(streaming=True) é inviável (primeira amostra
+# demora minutos para materializar).
+# ═══════════════════════════════════════════════════════════════════════════
+
+def _stream_via_rows_api(
+    dataset_name: str,
+    max_samples: int,
+    hf_token: Optional[str] = None,
+    batch_size: int = 100,
+    max_offset: int = 100_000,
+    config: Optional[str] = None,
+    split: str = "train",
+) -> Iterator[Dict[str, Any]]:
+    """Carrega amostras via HF datasets-server /rows API (HTTP).
+
+    Usado para datasets cujo parquet é muito grande para streaming eficiente
+    (ex: cnmoro/Instruct-PTBR-10M com 10 GB em arquivo único) OU datasets
+    que usam script Python como loader (não suportado em datasets 5.0+,
+    ex: adalbertojunior/punctuation-ptbr). A rows API retorna batches de
+    até 100 rows via HTTP, sem precisar baixar o parquet.
+
+    Args:
+        dataset_name: repo_id do dataset (ex: "cnmoro/Instruct-PTBR-10M")
+        max_samples: nº máximo de amostras a retornar
+        hf_token: token HF (opcional — dataset público)
+        batch_size: nº de rows por requisição (máx 100)
+        max_offset: offset máximo a tentar (sai do loop se excedido)
+        config: nome da config do dataset (default: "default"). NECESSÁRIO
+                para datasets cuja config não é "default" (ex:
+                adalbertojunior/punctuation-ptbr usa "punctuation-ptbr").
+        split: nome do split (default: "train").
+
+    Yields:
+        dict com campos do dataset (compatível com _normalize_sample)
+    """
+    import json as _json
+    import urllib.request
+    import urllib.parse
+
+    base_url = "https://datasets-server.huggingface.co/rows"
+    config_name = config or "default"
+    count = 0
+    offset = 0
+
+    while count < max_samples and offset < max_offset:
+        params = urllib.parse.urlencode({
+            "dataset": dataset_name,
+            "config": config_name,
+            "split": split,
+            "offset": offset,
+            "length": min(batch_size, max_samples - count),
+        })
+        url = f"{base_url}?{params}"
+        try:
+            req = urllib.request.Request(url)
+            if hf_token:
+                req.add_header("Authorization", f"Bearer {hf_token}")
+            with urllib.request.urlopen(req, timeout=30) as r:
+                data = _json.loads(r.read())
+        except Exception as e:
+            logger.warning(f"rows_api {dataset_name} config={config_name} offset={offset}: {e}")
+            break
+
+        rows = data.get("rows", [])
+        if not rows:
+            break
+
+        for row in rows:
+            if count >= max_samples:
+                return
+            row_data = row.get("row", {})
+            # Remove campos internos como __index_level_0__
+            clean = {k: v for k, v in row_data.items()
+                     if not k.startswith("__")}
+            yield clean
+            count += 1
+
+        offset += len(rows)
+        if len(rows) < batch_size:
+            break  # chegou ao fim do dataset
+
+    logger.info(f"rows_api {dataset_name} config={config_name}: streamou {count} amostras")
+
+
+def _stream_via_parquet_first_shard(
+    dataset_name: str,
+    max_samples: int,
+    hf_token: Optional[str] = None,
+) -> Iterator[Dict[str, Any]]:
+    """Baixa primeiro shard parquet do dataset e itera localmente com pyarrow.
+
+    Usado para datasets com múltiplos shards parquet grandes (ex:
+    strak2005/corpus-ptbr-v1 com 18 shards de ~1GB). Streaming via
+    datasets.load_dataset() não materializa primeira amostra em tempo
+    aceitável. Esta função baixa o primeiro shard (~26s para 1GB) e itera
+    localmente — rápido após download.
+
+    Args:
+        dataset_name: repo_id do dataset
+        max_samples: nº máximo de amostras
+        hf_token: token HF opcional
+
+    Yields:
+        dict com campos do dataset
+    """
+    from huggingface_hub import HfApi, hf_hub_download
+    import pyarrow.parquet as pq
+
+    api = HfApi(token=hf_token)
+
+    # Lista arquivos parquet no repo
+    try:
+        info = api.dataset_info(dataset_name)
+    except Exception as e:
+        logger.warning(f"parquet_shard {dataset_name}: dataset_info failed: {e}")
+        return
+
+    parquet_files = sorted([
+        s.rfilename for s in info.siblings
+        if s.rfilename.endswith(".parquet")
+    ])
+    if not parquet_files:
+        logger.warning(f"parquet_shard {dataset_name}: nenhum .parquet encontrado")
+        return
+
+    # Baixa apenas o primeiro shard
+    first_shard = parquet_files[0]
+    logger.info(f"parquet_shard {dataset_name}: baixando {first_shard}...")
+    try:
+        local_path = hf_hub_download(
+            repo_id=dataset_name,
+            filename=first_shard,
+            repo_type="dataset",
+            token=hf_token,
+        )
+    except Exception as e:
+        logger.warning(f"parquet_shard {dataset_name}: download failed: {e}")
+        return
+
+    # Itera com pyarrow.iter_batches — método eficiente que lê batches pequenos
+    # do parquet sem materializar o row group inteiro na memória.
+    # BUG FIX V13.9.2-finetune-v2: read_row_group() materializa TODAS as ~1M
+    # rows de uma vez (lento + ~1GB RAM). iter_batches(batch_size=10) lê só
+    # 10 rows por vez (rápido + ~10KB RAM).
+    try:
+        pf = pq.ParquetFile(local_path)
+        count = 0
+        # batch_read_size: nº de rows por batch (pequeno = menos RAM)
+        for batch in pf.iter_batches(batch_size=100):
+            if count >= max_samples:
+                break
+            cols = batch.schema.names
+            # to_pylist() é eficiente para batches pequenos
+            pydict = batch.to_pydict()
+            n_rows = len(pydict[cols[0]]) if cols else 0
+            for i in range(n_rows):
+                if count >= max_samples:
+                    break
+                row = {col: pydict[col][i] for col in cols}
+                yield row
+                count += 1
+    except Exception as e:
+        logger.warning(f"parquet_shard {dataset_name}: pyarrow iter failed: {e}")
+        return
+
+    logger.info(f"parquet_shard {dataset_name}: streamou {count} amostras do shard {first_shard}")
+
+
+@dataclass
+class ProcessedSample:
+    """Uma amostra processada: texto bruto + metadados."""
+    raw_text: str
+    dataset_name: str
+    sample_idx: int
+    text_field: str = ""
+    label_text: str = ""
+
+
+def _extract_field(sample: Dict[str, Any], candidate_fields: List[str]) -> Optional[str]:
+    """Extrai texto da primeira field disponível no sample.
+
+    Suporta:
+    - strings diretas
+    - listas de strings (join com espaço — útil para datasets tokenizados
+      como adalbertojunior/punctuation-ptbr onde "tokens": ["hoje","a",...])
+    - listas de dicts (chat messages — concatena role: content)
+    - dicts aninhados (pega text/content/body)
+    """
+    for field_name in candidate_fields:
+        if field_name in sample:
+            val = sample[field_name]
+            if val is None:
+                continue
+            if isinstance(val, str):
+                if val.strip():
+                    return val
+            elif isinstance(val, list):
+                # V6.5-7ds: se for lista de strings, join com espaços
+                # (ex: adalbertojunior/punctuation-ptbr tokens field)
+                string_parts = []
+                chat_parts = []
+                all_strings = True
+                for item in val:
+                    if isinstance(item, str):
+                        if item.strip():
+                            string_parts.append(item)
+                    elif isinstance(item, (int, float)):
+                        # tokens podem vir como ints (token IDs) — skip
+                        all_strings = False
+                        break
+                    elif isinstance(item, dict):
+                        all_strings = False
+                        role = item.get("role", "")
+                        content = item.get("content", "")
+                        if isinstance(content, str) and content.strip():
+                            chat_parts.append(f"{role}: {content}")
+                    else:
+                        all_strings = False
+                if all_strings and string_parts:
+                    joined = " ".join(string_parts).strip()
+                    if joined:
+                        return joined
+                if chat_parts:
+                    return "\n".join(chat_parts)
+            elif isinstance(val, dict):
+                # Pode ser nested
+                text = val.get("text") or val.get("content") or val.get("body")
+                if isinstance(text, str) and text.strip():
+                    return text
+    return None
+
+
+def _normalize_sample(sample: Dict[str, Any], dataset_name: str, idx: int) -> Optional[ProcessedSample]:
+    """Normaliza uma amostra bruta em ProcessedSample."""
+    fmt = DATASET_FORMATS.get(dataset_name, {})
+    text_fields = fmt.get("text_fields", ["text"])
+    label_fields = fmt.get("label_fields", [])
+
+    text = _extract_field(sample, text_fields)
+    if not text or len(text.strip()) < 10:  # skip very short
+        return None
+
+    label = ""
+    if label_fields:
+        label = _extract_field(sample, label_fields) or ""
+
+    # Se tem label, aplica template apropriado
+    template = fmt.get("format_template")
+    full_text = text
+
+    # V13.9.2-finetune-v3: para o template instruction_response, exige ambos
+    # os lados (instruction + response). Samples sem response são descartados
+    # para evitar treinar em texto parcial.
+    if template == "instruction_response" and not label:
+        return None
+
+    if label:
+        # V13.9.2-finetune-v3: format_template unificado "### Instruction:/### Response:"
+        # quando o dataset declarar explicitamente. Caso contrário, mantém o
+        # comportamento legado (concatenação simples text\nlabel) para os 9
+        # datasets base — retrocompatibilidade total.
+        if template == "instruction_response":
+            instr_prefix = fmt.get("instruction_prefix", "")
+            if instr_prefix:
+                full_text = (
+                    f"### Instruction:\n{instr_prefix}\n{text}\n\n"
+                    f"### Response:\n{label}"
+                )
+            else:
+                full_text = (
+                    f"### Instruction:\n{text}\n\n"
+                    f"### Response:\n{label}"
+                )
+        elif template == "title_text_concat":
+            # V6.5-7ds: para datasets de notícias (iara-project, manoela),
+            # o campo "text" (corpo da notícia) é o conteúdo principal e
+            # "category" (categoria) é o contexto temático. Formatamos como
+            # "TEXTO: <corpo>\nCATEGORIA: <categoria>".
+            full_text = f"TEXTO: {text}\nCATEGORIA: {label}"
+        else:
+            # Comportamento legado (9 datasets base): concatenação simples
+            full_text = f"{text}\n{label}"
+
+    return ProcessedSample(
+        raw_text=full_text,
+        dataset_name=dataset_name,
+        sample_idx=idx,
+        text_field=text_fields[0] if text_fields else "",
+        label_text=label,
+    )
+
+
+def _load_dataset_streaming(
+    dataset_name: str,
+    split: str = "train",
+    config: Optional[str] = None,
+    hf_token: Optional[str] = None,
+):
+    """Carrega dataset em modo streaming.
+
+    BUG FIX V13.9.1: Removido trust_remote_code (deprecated).
+    Se o dataset não carregar com split especificado, tenta 'train' como fallback.
+    BUG FIX V13.9.2: Adicionado fallback para datasets com script (carolina-c4ai).
+    """
+    from datasets import load_dataset
+
+    # Lista de splits para tentar, em ordem
+    splits_to_try = [split]
+    if split != "train":
+        splits_to_try.append("train")
+    # Adiciona outros splits comuns
+    for s in ["test", "validation", "cot", "tir"]:
+        if s not in splits_to_try:
+            splits_to_try.append(s)
+
+    fmt = DATASET_FORMATS.get(dataset_name, {})
+    fallback_name = fmt.get("fallback")
+    datasets_to_try = [dataset_name]
+    if fallback_name:
+        datasets_to_try.append(fallback_name)
+
+    for try_ds_name in datasets_to_try:
+        for try_split in splits_to_try:
+            try:
+                if config:
+                    ds = load_dataset(
+                        try_ds_name, config, split=try_split, streaming=True,
+                        token=hf_token,
+                    )
+                else:
+                    ds = load_dataset(
+                        try_ds_name, split=try_split, streaming=True,
+                        token=hf_token,
+                    )
+                logger.info(f"Loaded {try_ds_name} split={try_split}")
+                return ds
+            except Exception as e:
+                err_str = str(e).lower()
+                # If "Bad split" error, try next split
+                if "bad split" in err_str or "available splits" in err_str:
+                    logger.info(f"  Split {try_split} not available for {try_ds_name}, trying next")
+                    continue
+                # If "scripts no longer supported", try next dataset (fallback)
+                if "scripts are no longer supported" in err_str and fallback_name:
+                    logger.info(f"  {try_ds_name} uses script — trying fallback {fallback_name}")
+                    break
+                # For other errors, log and try next split
+                logger.warning(f"Failed to load {try_ds_name} split={try_split}: {str(e)[:120]}")
+                continue
+
+    logger.warning(f"All split attempts failed for {dataset_name} (will be skipped)")
+    return None
+
+
+def stream_dataset(
+    dataset_name: str,
+    max_samples: int = 500,
+    hf_token: Optional[str] = None,
+) -> Iterator[ProcessedSample]:
+    """Faz streaming de um dataset, retornando até max_samples amostras.
+
+    Args:
+        dataset_name: nome do dataset no HuggingFace
+        max_samples: número máximo de amostras a retornar
+        hf_token: token HF opcional
+
+    Yields:
+        ProcessedSample
+    """
+    fmt = DATASET_FORMATS.get(dataset_name)
+    if fmt is None:
+        logger.error(f"Unknown dataset: {dataset_name}")
+        return
+
+    # ── V13.9.2-carolina: bypass do script Python ─────────────────────────
+    # O dataset carolina-c4ai/corpus-carolina usa um script Python como loader,
+    # que não é suportado por datasets 5.0+. Streamamos os XML.gz diretamente.
+    if dataset_name == "carolina-c4ai/corpus-carolina":
+        logger.info(f"Carolina: usando carregador direto (bypass do script Python)")
+        count = 0
+        try:
+            for raw_sample in _stream_carolina_direct(
+                max_samples=max_samples, hf_token=hf_token
+            ):
+                processed = _normalize_sample(raw_sample, dataset_name, count)
+                if processed is not None:
+                    yield processed
+                    count += 1
+        except Exception as e:
+            logger.warning(f"Carolina direct stream error at sample {count}: {e}")
+            return
+        logger.info(f"Streamed {count} samples from {dataset_name} (direct XML)")
+        return
+
+    # ── V13.9.2-finetune-v2: loaders alternativos para parquets grandes ───
+    # Alguns datasets (cnmoro, strak2005/bratao) têm parquets tão grandes que
+    # datasets.load_dataset(streaming=True) demora minutos para materializar
+    # a primeira amostra. Usamos loaders alternativos especificados no campo
+    # "loader" do formato.
+    loader = fmt.get("loader")
+    if loader == "rows_api":
+        logger.info(f"{dataset_name}: usando rows_api loader (datasets-server HTTP)")
+        count = 0
+        try:
+            for raw_sample in _stream_via_rows_api(
+                dataset_name, max_samples=max_samples, hf_token=hf_token,
+                config=fmt.get("config"),
+                split=fmt.get("split", "train"),
+            ):
+                processed = _normalize_sample(raw_sample, dataset_name, count)
+                if processed is not None:
+                    yield processed
+                    count += 1
+        except Exception as e:
+            logger.warning(f"rows_api stream error at sample {count}: {e}")
+            return
+        logger.info(f"Streamed {count} samples from {dataset_name} (rows_api)")
+        return
+
+    if loader == "parquet_first_shard":
+        logger.info(f"{dataset_name}: usando parquet_first_shard loader")
+        count = 0
+        try:
+            for raw_sample in _stream_via_parquet_first_shard(
+                dataset_name, max_samples=max_samples, hf_token=hf_token
+            ):
+                processed = _normalize_sample(raw_sample, dataset_name, count)
+                if processed is not None:
+                    yield processed
+                    count += 1
+        except Exception as e:
+            logger.warning(f"parquet_shard stream error at sample {count}: {e}")
+            # Tenta fallback (ex: strak2005 -> bratao)
+            fallback_name = fmt.get("fallback")
+            if fallback_name and fallback_name != dataset_name:
+                logger.info(f"Tentando fallback: {fallback_name}")
+                fallback_fmt = DATASET_FORMATS.get(fallback_name, {})
+                # Recursão respeitando o loader do fallback
+                yield from stream_dataset(
+                    fallback_name, max_samples=max_samples, hf_token=hf_token
+                )
+            return
+        logger.info(f"Streamed {count} samples from {dataset_name} (parquet_first_shard)")
+        return
+
+    ds = _load_dataset_streaming(
+        dataset_name,
+        split=fmt.get("split", "train"),
+        config=fmt.get("config"),
+        hf_token=hf_token,
+    )
+    if ds is None:
+        return
+
+    count = 0
+    try:
+        for idx, sample in enumerate(ds):
+            if count >= max_samples:
+                break
+            processed = _normalize_sample(sample, dataset_name, idx)
+            if processed is not None:
+                yield processed
+                count += 1
+    except Exception as e:
+        logger.warning(f"Error streaming {dataset_name} at sample {idx}: {e}")
+        return
+
+    logger.info(f"Streamed {count} samples from {dataset_name}")
+
+
+def stream_all_datasets(
+    datasets: Optional[List[str]] = None,
+    max_samples_per_dataset: int = 500,
+    hf_token: Optional[str] = None,
+) -> Iterator[ProcessedSample]:
+    """Faz streaming sequencial de múltiplos datasets.
+
+    Args:
+        datasets: lista de nomes (default: DEFAULT_DATASETS)
+        max_samples_per_dataset: limite por dataset
+        hf_token: token HF opcional
+
+    Yields:
+        ProcessedSample
+    """
+    if datasets is None:
+        datasets = DEFAULT_DATASETS
+
+    for ds_name in datasets:
+        logger.info(f"Starting dataset: {ds_name}")
+        yield from stream_dataset(
+            ds_name,
+            max_samples=max_samples_per_dataset,
+            hf_token=hf_token,
+        )
+
+
+def collate_samples_to_tensors(
+    samples: List[ProcessedSample],
+    tokenizer,
+    max_seq_len: int = 64,
+    pad_token_id: int = 0,
+) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
+    """Converte lista de ProcessedSample em tensors para treino.
+
+    BUG FIX V13.9.1: Tokenizer.encode() retorna 'Encoding' object, não list.
+    Use .ids attribute to get token IDs.
+
+    Returns:
+        input_ids: (B, T)
+        attention_mask: (B, T)
+        labels: (B, T) — same as input_ids (LM training)
+    """
+    batch_input_ids = []
+    batch_attn_mask = []
+
+    for sample in samples:
+        # Tokenize
+        encoding = tokenizer.encode(sample.raw_text, add_special_tokens=True)
+
+        # BUG FIX: Handle both `tokenizers.Encoding` and `transformers` tokenizer outputs
+        if hasattr(encoding, 'ids'):
+            tokens = encoding.ids  # tokenizers library
+        elif isinstance(encoding, list):
+            tokens = encoding  # already a list of ints
+        elif hasattr(encoding, 'input_ids'):
+            tokens = encoding.input_ids  # transformers BatchEncoding
+        else:
+            tokens = list(encoding)  # fallback
+
+        # Truncate to max_seq_len
+        if len(tokens) > max_seq_len:
+            tokens = tokens[:max_seq_len]
+        # Pad
+        attn_mask = [1] * len(tokens) + [0] * (max_seq_len - len(tokens))
+        tokens = list(tokens) + [pad_token_id] * (max_seq_len - len(tokens))
+        batch_input_ids.append(tokens)
+        batch_attn_mask.append(attn_mask)
+
+    input_ids = torch.tensor(batch_input_ids, dtype=torch.long)
+    attention_mask = torch.tensor(batch_attn_mask, dtype=torch.long)
+    labels = input_ids.clone()
+
+    return input_ids, attention_mask, labels
+
+
+# ═══════════════════════════════════════════════════════════════════════════
+# Self-test (NÃO enviar para HuggingFace)
+# ═══════════════════════════════════════════════════════════════════════════
+if __name__ == "__main__":
+    print("=== Streaming Datasets V13.9.1 Self-Test ===\n")
+
+    # Test 1: Test field extraction
+    print("Test 1: Field extraction")
+    sample_chat = {
+        "messages": [
+            {"role": "user", "content": "Olá"},
+            {"role": "assistant", "content": "Como posso ajudar?"},
+        ]
+    }
+    text = _extract_field(sample_chat, ["messages", "text"])
+    print(f"  Chat text: {text!r}")
+    assert text and "Olá" in text, "Field extraction failed for chat"
+
+    sample_text = {"text": "Hello world"}
+    text = _extract_field(sample_text, ["text", "content"])
+    assert text == "Hello world"
+    print(f"  Text field: {text!r}")
+    print(f"  OK\n")
+
+    # Test 2: Test list of datasets
+    print("Test 2: Dataset list")
+    print(f"  Configured datasets ({len(DEFAULT_DATASETS)}):")
+    for i, ds in enumerate(DEFAULT_DATASETS, 1):
+        print(f"    {i}. {ds}")
+    assert len(DEFAULT_DATASETS) == 14
+    # V6.5-V2: verifica sequência CONHECIMENTO (8 datasets) + PUNICAO (1)
+    assert len(CONHECIMENTO_DATASETS_V2) == 8
+    assert PUNICAO_DATASET_V2 == "BrunoN-Dev/corpus-ptbr-v1"
+    print(f"  V2 CONHECIMENTO datasets: {len(CONHECIMENTO_DATASETS_V2)}")
+    print(f"  V2 PUNICAO dataset: {PUNICAO_DATASET_V2}")
+    print(f"  OK\n")
+
+    print("=== ALL STREAMING TESTS PASSED ===")

v6_5_v2_attention_eval.json

@@ -3,18 +3,18 @@
   "user_requirement": "verificar se o mecanismo de atenção está ativo e acessado logicamente funcional",
   "metrics": {
     "active": true,
-    "n_calls": 10680,
+    "n_calls": 10136,
     "n_errors": 0,
-    "last_norm_in": 111.9294204711914,
-    "last_norm_out": 116.00782012939453,
+    "last_norm_in": 78.78106689453125,
+    "last_norm_out": 83.25476837158203,
     "last_attn_activated": true,
-    "last_attn_diff_norm": 116.50959014892578,
+    "last_attn_diff_norm": 81.70692443847656,
     "n_heads": 8,
     "logic_functional": true
   },
   "active": true,
   "logic_functional": true,
-  "n_calls": 10680,
+  "n_calls": 10136,
   "n_errors": 0,
   "n_heads": 8,
   "assessment": "PASS"

v6_5_v2_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e6ec0c41f5e815e0451cd8627febd2dbe76395a1e0e3cb4a11cb58f98ddb29fe
+size 56004222

v6_5_v2_model_states_after_conhecimento.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1b943657505dfb8f42b12875272a1395932d836755aea0cf52ba1c4ce7a901d2
+size 53528857

v6_5_v2_report.json

@@ -1,6 +1,6 @@
 {
   "version": "V6.5-V2",
-  "timestamp": "2026-08-08T23:52:08.862775",
+  "timestamp": "2026-08-09T01:05:21.476394",
   "config": {
     "som_grid": [
       6,
@@ -9,9 +9,9 @@
       4
     ],
     "n_neurons": 864,
-    "hidden_dim": 1024,
-    "vocab_size": 16384,
-    "n_hypotheses": 16,
+    "hidden_dim": 512,
+    "vocab_size": 8192,
+    "n_hypotheses": 8,
     "n_trials": 3,
     "hyp_train_steps": 30,
     "stream_batch_size": 100,
@@ -54,10 +54,10 @@
     "init_done": true
   },
   "fp16_benchmark": {
-    "best_time_ms": 91.0796530006337,
-    "avg_time_ms": 91.46852800040506,
-    "best_tflops": 1.4053632812930177,
-    "avg_tflops": 1.3993884322641899,
+    "best_time_ms": 85.25625100082834,
+    "avg_time_ms": 97.09998300058942,
+    "best_tflops": 1.5013561879322652,
+    "avg_tflops": 1.3182288610619326,
     "matrix_size": 4000.0
   },
   "v2_verification": {
@@ -86,13 +86,13 @@
   "fase_1_conhecimento_summary": {
     "total_samples": 8000,
     "meta_atingida": true,
-    "elapsed_s": 722.1084952354431,
+    "elapsed_s": 337.3461060523987,
     "storage_critical_stopped": false
   },
   "fase_2_punicão_summary": {
     "total_samples": 2000,
     "meta_atingida": true,
-    "elapsed_s": 507.18996810913086,
+    "elapsed_s": 221.32905673980713,
     "punishment_events": 120,
     "hypotheses_trainings": 60,
     "delta_applications": 60,
@@ -101,7 +101,7 @@
   "attention_eval_summary": {
     "active": true,
     "logic_functional": true,
-    "n_calls": 10680,
+    "n_calls": 10136,
     "assessment": "PASS"
   },
   "predict_fix_summary": {
@@ -117,36 +117,41 @@
     "n_with_think": 3,
     "answer_rate": 1.0,
     "think_rate": 1.0,
-    "avg_latency_ms": 11.263132095336914,
+    "avg_latency_ms": 6.689548492431641,
     "avg_reasoning_length": 1026.0
   },
   "model_states_saved_to": "/home/z/my-project/BiGRU_T_version/v6_5_v2_model_states.pt",
   "save_info": {
     "saved": true,
     "path": "/home/z/my-project/BiGRU_T_version/v6_5_v2_model_states.pt",
-    "size_mb": 220.174132,
+    "size_mb": 56.004222,
+    "size_gb": 0.05215799622237682,
+    "size_status": "OK",
+    "size_within_1gb_limit": true,
     "reason": "end_of_training_v2",
     "step": 700,
     "total_samples": 10000,
     "n_tensors": 7,
-    "n_buffer_tail": 128
+    "n_buffer_tail": 64
   },
   "final_v2_metrics": {
     "version": "V2-dynamic",
-    "n_hypotheses": 16,
-    "n_hypotheses_active": 16,
-    "max_n_hypotheses": 32,
+    "n_hypotheses": 8,
+    "n_hypotheses_active": 8,
+    "max_n_hypotheses": 16,
     "n_trials": 6,
     "hyp_train_steps": 30,
     "hyp_lr": 0.0001,
     "delta_scale": 0.0010000000474974513,
-    "n_generators": 32,
+    "n_generators": 16,
     "punishment_count": 0,
     "success_count": 0,
     "training_ready": false,
     "classifier_trained": true,
-    "ewc_reference_set": true,
+    "ewc_reference_set": false,
     "buffer_size": 256,
+    "total_hyp_steps_executed": 1800,
+    "n_train_hyp_calls": 60,
     "dynamic_adaptation": {
       "loss_history_len": 8,
       "loss_stats": {
@@ -161,7 +166,7 @@
       "n_adaptations": 20,
       "limits": {
         "min_n_hypotheses": 4,
-        "max_n_hypotheses": 32,
+        "max_n_hypotheses": 16,
         "min_n_trials": 1,
         "max_n_trials": 6,
         "min_hyp_train_steps": 10,
@@ -172,12 +177,12 @@
           "trigger": "punishment",
           "step": 115,
           "before": {
-            "n_hypotheses": 16,
+            "n_hypotheses": 8,
             "n_trials": 6,
             "hyp_train_steps": 30
           },
           "after": {
-            "n_hypotheses": 16,
+            "n_hypotheses": 8,
             "n_trials": 6,
             "hyp_train_steps": 30
           },
@@ -196,12 +201,12 @@
           "trigger": "auto",
           "step": 116,
           "before": {
-            "n_hypotheses": 16,
+            "n_hypotheses": 8,
             "n_trials": 6,
             "hyp_train_steps": 30
           },
           "after": {
-            "n_hypotheses": 16,
+            "n_hypotheses": 8,
             "n_trials": 6,
             "hyp_train_steps": 30
           },
@@ -220,12 +225,12 @@
           "trigger": "punishment",
           "step": 117,
           "before": {
-            "n_hypotheses": 16,
+            "n_hypotheses": 8,
             "n_trials": 6,
             "hyp_train_steps": 30
           },
           "after": {
-            "n_hypotheses": 16,
+            "n_hypotheses": 8,
             "n_trials": 6,
             "hyp_train_steps": 30
           },
@@ -244,12 +249,12 @@
           "trigger": "auto",
           "step": 118,
           "before": {
-            "n_hypotheses": 16,
+            "n_hypotheses": 8,
             "n_trials": 6,
             "hyp_train_steps": 30
           },
           "after": {
-            "n_hypotheses": 16,
+            "n_hypotheses": 8,
             "n_trials": 6,
             "hyp_train_steps": 30
           },
@@ -268,12 +273,12 @@
           "trigger": "punishment",
           "step": 119,
           "before": {
-            "n_hypotheses": 16,
+            "n_hypotheses": 8,
             "n_trials": 6,
             "hyp_train_steps": 30
           },
           "after": {
-            "n_hypotheses": 16,
+            "n_hypotheses": 8,
             "n_trials": 6,
             "hyp_train_steps": 30
           },

v6_5_v2_user_questions.json

@@ -25,10 +25,10 @@
       "has_answer": true,
       "has_decompose": true,
       "think_preview": "Analisando a query: 'Luva de Pedreiro Távila'\nIdentificando o tipo de problema e requisitos.\nDeterminando se ferramentas são necessárias.",
-      "answer_preview": "prediction=short_text | BMU=(0, 0, 0, 0)",
+      "answer_preview": "prediction=short_text | BMU=(2, 5, 5, 1)",
       "raw_response_preview": "<think>\nAnalisando a query: 'Luva de Pedreiro Távila'\nIdentificando o tipo de problema e requisitos.\nDeterminando se ferramentas são necessárias.\n</think>\n<plan>\nPlano de resolução:\n1. Decompor o problema em sub-tarefas\n2. Identificar ferramentas necessárias (disponíveis: som_query, buffer_stats)\n3. Executar sub-tarefas em sequência\n4. Monitorar resultados\n5. Compor resposta final\n</plan>\n<decompose>\n- Processar: Luva de Pedreiro Távila\n</decompose>\n<execute>\nSub-tarefa 'Processar: Luva de Pedre...",
       "n_tags": 4,
-      "latency_ms": 9.467601776123047
+      "latency_ms": 7.518291473388672
     },
     {
       "query": "Lula reserva valor",
@@ -43,10 +43,10 @@
       "has_answer": true,
       "has_decompose": true,
       "think_preview": "Analisando a query: 'Lula reserva valor'\nIdentificando o tipo de problema e requisitos.\nDeterminando se ferramentas são necessárias.",
-      "answer_preview": "prediction=short_text | BMU=(0, 0, 0, 0)",
+      "answer_preview": "prediction=short_text | BMU=(2, 5, 5, 1)",
       "raw_response_preview": "<think>\nAnalisando a query: 'Lula reserva valor'\nIdentificando o tipo de problema e requisitos.\nDeterminando se ferramentas são necessárias.\n</think>\n<plan>\nPlano de resolução:\n1. Decompor o problema em sub-tarefas\n2. Identificar ferramentas necessárias (disponíveis: som_query, buffer_stats)\n3. Executar sub-tarefas em sequência\n4. Monitorar resultados\n5. Compor resposta final\n</plan>\n<decompose>\n- Processar: Lula reserva valor\n</decompose>\n<execute>\nSub-tarefa 'Processar: Lula reserva valor' exe...",
       "n_tags": 4,
-      "latency_ms": 11.906862258911133
+      "latency_ms": 6.790876388549805
     },
     {
       "query": "Amazonas força-tarefa vítimas",
@@ -61,10 +61,10 @@
       "has_answer": true,
       "has_decompose": true,
       "think_preview": "Analisando a query: 'Amazonas força-tarefa vítimas'\nIdentificando o tipo de problema e requisitos.\nDeterminando se ferramentas são necessárias.",
-      "answer_preview": "prediction=short_text | BMU=(0, 0, 0, 0)",
+      "answer_preview": "prediction=short_text | BMU=(2, 5, 5, 1)",
       "raw_response_preview": "<think>\nAnalisando a query: 'Amazonas força-tarefa vítimas'\nIdentificando o tipo de problema e requisitos.\nDeterminando se ferramentas são necessárias.\n</think>\n<plan>\nPlano de resolução:\n1. Decompor o problema em sub-tarefas\n2. Identificar ferramentas necessárias (disponíveis: som_query, buffer_stats)\n3. Executar sub-tarefas em sequência\n4. Monitorar resultados\n5. Compor resposta final\n</plan>\n<decompose>\n- Processar: Amazonas força-tarefa vítimas\n</decompose>\n<execute>\nSub-tarefa 'Processar: A...",
       "n_tags": 4,
-      "latency_ms": 12.414932250976562
+      "latency_ms": 5.759477615356445
     }
   ],
   "summary": {
@@ -72,7 +72,7 @@
     "n_with_think": 3,
     "answer_rate": 1.0,
     "think_rate": 1.0,
-    "avg_latency_ms": 11.263132095336914,
+    "avg_latency_ms": 6.689548492431641,
     "avg_reasoning_length": 1026.0
   },
   "quality_assessment": {

vqvae2_hierarchical.py

@@ -0,0 +1,200 @@
+"""
+Xavante - vqvae2_hierarchical.py
+Responsabilidade: VQ-VAE-2 hierarquica (Teorema 13.1/13.2/13.3).
+Reutiliza implementacao validada do FlexNet com EMA + Goose VQ + dead code restart.
+Referencia: Razavi et al. 2019 - Generating Diverse High-Fidelity Images with VQ-VAE-2.
+"""
+from __future__ import annotations
+
+import logging
+import math
+from typing import List, Tuple
+
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+
+logger = logging.getLogger(__name__)
+
+
+class VectorQuantizerEMA(nn.Module):
+    """
+    VQ com atualizacao EMA do codebook (validado no FlexNet).
+    Inclui: dead code restart, commitment loss, Goose VQ.
+    v6 aprimoramentos:
+    - Diversity loss (entropy bonus) para forcar uso uniforme do codebook
+    - Commitment warmup: beta_t = beta_0 * min(1, t/warmup)
+    - EMA restart mais agressivo (decay adaptativo)
+    """
+
+    def __init__(
+        self,
+        num_embeddings: int,
+        embedding_dim: int,
+        commitment_cost: float = 0.25,
+        decay: float = 0.99,
+        epsilon: float = 1e-5,
+        dead_threshold: int = 1,
+        diversity_weight: float = 0.1,
+        warmup_steps: int = 500,
+    ):
+        super().__init__()
+        self.num_embeddings = num_embeddings
+        self.embedding_dim = embedding_dim
+        self.commitment_cost = commitment_cost
+        self.decay = decay
+        self.epsilon = epsilon
+        self.dead_threshold = dead_threshold
+        self.diversity_weight = diversity_weight
+        self.warmup_steps = warmup_steps
+        self._step = 0
+
+        embed = torch.empty(num_embeddings, embedding_dim)
+        nn.init.normal_(embed, mean=0.0, std=0.02)
+        self.register_buffer("embeddings", embed)
+        self.register_buffer("ema_count", torch.zeros(num_embeddings))
+        self.register_buffer("ema_weight", embed.clone())
+        self.register_buffer("usage_count", torch.zeros(num_embeddings, dtype=torch.long))
+
+    def forward(self, x: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
+        """
+        x: [B, D, H, W] ou [B, D]
+        Retorna (quantized, loss, encoding_indices)
+        """
+        if x.dim() == 4:
+            # [B, D, H, W] -> [B, H, W, D] -> [B*H*W, D]
+            B, D, H, W = x.shape
+            flat = x.permute(0, 2, 3, 1).reshape(-1, self.embedding_dim)
+        else:
+            # [B, D] -> [B, D]
+            flat = x.reshape(-1, self.embedding_dim)
+        # Distancia
+        distances = (
+            flat.pow(2).sum(1, keepdim=True)
+            - 2 * flat @ self.embeddings.t()
+            + self.embeddings.pow(2).sum(1)
+        )
+        indices = torch.argmin(distances, dim=1)
+        one_hot = F.one_hot(indices, self.num_embeddings).type(flat.dtype)
+        quantized = one_hot @ self.embeddings
+        # Uso
+        self.usage_count.index_add_(0, indices, torch.ones_like(indices))
+
+        # EMA update
+        if self.training:
+            with torch.no_grad():
+                count = one_hot.sum(0)
+                ema_count_new = self.decay * self.ema_count + (1 - self.decay) * count
+                total = ema_count_new.sum()
+                ema_count_new = (ema_count_new + self.epsilon) / (total + self.num_embeddings * self.epsilon) * total
+                dw = one_hot.t() @ flat
+                self.ema_weight = self.decay * self.ema_weight + (1 - self.decay) * dw
+                self.ema_count = ema_count_new
+                self.embeddings.copy_(self.ema_weight / self.ema_count.unsqueeze(1))
+
+                # Dead code restart
+                dead = self.usage_count < self.dead_threshold
+                if dead.any() and flat.shape[0] > 0:
+                    dead_indices = dead.nonzero(as_tuple=True)[0]
+                    n_dead = dead_indices.shape[0]
+                    # Sample min(n_dead, flat.shape[0]) candidates from flat
+                    n_replace = min(n_dead, flat.shape[0])
+                    if n_replace > 0:
+                        idx = torch.randperm(flat.shape[0])[:n_replace].to(flat.device)
+                        # Substituir apenas n_replace codigos mortos
+                        self.embeddings.data[dead_indices[:n_replace]] = flat[idx].detach()
+
+        # Commitment loss com warmup
+        self._step += 1
+        beta = self.commitment_cost * min(1.0, self._step / max(self.warmup_steps, 1))
+        e_latent_loss = F.mse_loss(quantized.detach(), flat)
+        q_latent_loss = F.mse_loss(quantized, flat.detach())
+        # Diversity loss: -H(p) onde p = freq de uso de cada code
+        # Maximizar entropia = forçar uso uniforme do codebook
+        if self.diversity_weight > 0 and self.training:
+            code_probs = one_hot.float().mean(dim=0)  # [num_embeddings]
+            code_probs = code_probs.clamp_min(1e-10)
+            entropy = -(code_probs * code_probs.log()).sum()
+            diversity_loss = -entropy / math.log(self.num_embeddings)  # normalize to [0, 1]
+        else:
+            diversity_loss = torch.tensor(0.0, device=x.device)
+        loss = q_latent_loss + beta * e_latent_loss + self.diversity_weight * diversity_loss
+
+        # Straight-through
+        quantized = flat + (quantized - flat).detach()
+        # Restore shape
+        if x.dim() == 4:
+            quantized = quantized.view(B, H, W, D).permute(0, 3, 1, 2).contiguous()
+            indices = indices.view(B, H, W)
+        else:
+            quantized = quantized.view_as(x)
+        return quantized, loss, indices
+
+
+class HierarchicalVQVAE2(nn.Module):
+    """
+    VQ-VAE-2 com 2 niveis: top (global) e bottom (local).
+    Mapeia: Teorema 13.1/13.2/13.3.
+    """
+
+    def __init__(
+        self,
+        in_channels: int = 3,
+        hidden_dim: int = 64,
+        top_codebook: int = 128,
+        bottom_codebook: int = 512,
+        commitment_cost: float = 0.25,
+        decay: float = 0.99,
+    ):
+        super().__init__()
+        self.encoder_bottom = nn.Sequential(
+            nn.Conv2d(in_channels, hidden_dim, 4, 2, 1),
+            nn.GELU(),
+            nn.Conv2d(hidden_dim, hidden_dim, 4, 2, 1),
+            nn.GELU(),
+            nn.Conv2d(hidden_dim, hidden_dim, 3, 1, 1),
+        )
+        self.encoder_top = nn.Sequential(
+            nn.Conv2d(hidden_dim, hidden_dim, 4, 2, 1),
+            nn.GELU(),
+            nn.Conv2d(hidden_dim, hidden_dim, 4, 2, 1),
+            nn.GELU(),
+            nn.Conv2d(hidden_dim, hidden_dim, 3, 1, 1),
+        )
+        self.vq_top = VectorQuantizerEMA(top_codebook, hidden_dim, commitment_cost, decay)
+        self.vq_bottom = VectorQuantizerEMA(bottom_codebook, hidden_dim, commitment_cost, decay)
+        self.decoder_top = nn.Sequential(
+            nn.ConvTranspose2d(hidden_dim, hidden_dim, 4, 2, 1),
+            nn.GELU(),
+            nn.ConvTranspose2d(hidden_dim, hidden_dim, 4, 2, 1),
+            nn.GELU(),
+        )
+        self.decoder_bottom = nn.Sequential(
+            nn.ConvTranspose2d(hidden_dim * 2, hidden_dim, 4, 2, 1),
+            nn.GELU(),
+            nn.ConvTranspose2d(hidden_dim, hidden_dim, 4, 2, 1),
+            nn.GELU(),
+            nn.Conv2d(hidden_dim, in_channels, 3, 1, 1),
+        )
+
+    def forward(self, x: torch.Tensor) -> dict:
+        # Bottom encode
+        ze_b = self.encoder_bottom(x)
+        # Top encode
+        ze_t = self.encoder_top(ze_b)
+        zq_t, loss_t, idx_t = self.vq_top(ze_t)
+        # Decodifica top e soma com bottom
+        dec_t = self.decoder_top(zq_t)
+        # Concat com bottom quantized
+        zq_b, loss_b, idx_b = self.vq_bottom(ze_b)
+        cat = torch.cat([dec_t, zq_b], dim=1)
+        x_recon = self.decoder_bottom(cat)
+        return {
+            "reconstruction": x_recon,
+            "loss": loss_t + loss_b,
+            "top_indices": idx_t,
+            "bottom_indices": idx_b,
+        }
+
+
+__all__ = ["VectorQuantizerEMA", "HierarchicalVQVAE2"]

vqvae2_hierarchical_flexnet.py

@@ -0,0 +1,513 @@
+"""
+vqvae2_hierarchical — VQ-VAE-2 (Razavi 2019) hierárquico com 2 níveis de codebook.
+
+Arquitetura:
+- Top level: codebook K_top captura estrutura global (low-resolution)
+- Bottom level: codebook K_bot captura detalhes locais (high-resolution)
+- Cada nível tem encoder/decoder proprio
+- Residual quantization: bottom level quantiza o RESÍDUO do top level
+
+Loss total:
+    L = ||x - D(top_q, bot_q)||²
+        + ||sg[z_top] - top_q||² + β·||z_top - sg[top_q]||²
+        + ||sg[z_bot] - bot_q||² + β·||z_bot - sg[bot_q]||²
+
+Implementa:
+- EMA updates em ambos codebooks
+- Dead code restart em ambos codebooks
+- Goose VQ com schedule de temperatura (alto→baixo)
+
+Setup dos testes (solicitado):
+- batch=256, K_top=512, K_bot=512
+- Goose τ schedule: começa 2.0, decai para 0.5 via cosine
+- EMA decay γ=0.99
+- Dead code restart a cada 3 épocas
+"""
+
+from __future__ import annotations
+from typing import Dict, Optional, Tuple
+
+import math
+import numpy as np
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+
+
+# ============================================================
+# v12.8: Normalizações alternativas (substituem LayerNorm v12.7)
+# ============================================================
+
+class RMSNorm(nn.Module):
+    """RMSNorm — preserva magnitude (não subtrai média como LayerNorm).
+
+    y = x / sqrt(mean(x²) + eps) * gamma
+
+    Vantagens sobre LayerNorm:
+    - Mais barato (sem cálculo de variância)
+    - Preserva magnitude do sinal (importante para VQ-VAE)
+    - Zhang & Sennrich 2019
+    """
+
+    def __init__(self, dim: int, eps: float = 1e-6):
+        super().__init__()
+        self.gamma = nn.Parameter(torch.ones(dim))
+        self.eps = eps
+
+    def forward(self, x: torch.Tensor) -> torch.Tensor:
+        rms = torch.sqrt(x.pow(2).mean(dim=-1, keepdim=True) + self.eps)
+        return self.gamma * x / rms
+
+
+def apply_weightnorm(module: nn.Module) -> nn.Module:
+    """Aplica weight normalization (EnCodec-style) a um módulo Linear.
+
+    Salakhutdinov & Wang 2016: w = g * v / ||v||, onde g é escalar aprendido.
+
+    Vantagens:
+    - Normaliza apenas o peso, não a ativação
+    - Preserva estatística do input (importante para VQ-VAE)
+    - Usado em EnCodec (Défossez 2022) e SoundStream (Zeghidour 2021)
+    """
+    return nn.utils.weight_norm(module)
+
+
+class HierarchicalVectorQuantizer(nn.Module):
+    """Quantizador hierárquico com 2 níveis (top + bottom) e técnicas anti-collapse.
+
+    Args:
+        num_codes_top: K_top (codebook de topo, captura estrutura global)
+        num_codes_bot: K_bot (codebook de base, captura detalhes)
+        code_dim: D (dimensão compartilhada)
+        beta: peso do commitment loss
+        ema_decay: γ para EMA (default 0.99)
+        dead_code_threshold: N_k abaixo do qual código é morto
+        dead_code_restart_every: a cada quantas épocas verificar
+        goose_temp_init: temperatura Goose inicial (default 2.0)
+        goose_temp_final: temperatura Goose final (default 0.5)
+        goose_schedule: 'cosine' ou 'linear'
+        total_epochs: total de épocas (para schedule)
+    """
+
+    def __init__(
+        self,
+        num_codes_top: int = 512,
+        num_codes_bot: int = 512,
+        code_dim: int = 32,
+        beta: float = 0.25,
+        ema_decay: float = 0.99,
+        dead_code_threshold: float = 1.0,
+        dead_code_restart_every: int = 3,
+        goose_temp_init: float = 2.0,
+        goose_temp_final: float = 0.5,
+        goose_schedule: str = "cosine",
+        total_epochs: int = 25,
+        norm_type: str = "none",  # v12.8: "none" | "weightnorm" | "rmsnorm"
+    ):
+        super().__init__()
+        self.num_codes_top = num_codes_top
+        self.num_codes_bot = num_codes_bot
+        self.code_dim = code_dim
+        self.beta = beta
+        self.ema_decay = ema_decay
+        self.dead_code_threshold = dead_code_threshold
+        self.dead_code_restart_every = dead_code_restart_every
+        self.goose_temp_init = goose_temp_init
+        self.goose_temp_final = goose_temp_final
+        self.goose_schedule = goose_schedule
+        self.total_epochs = total_epochs
+        self.norm_type = norm_type  # v12.8
+
+        # Codebooks (buffers, atualizados via EMA).
+        self.register_buffer('codebook_top', torch.randn(num_codes_top, code_dim) * 0.05)
+        self.register_buffer('codebook_bot', torch.randn(num_codes_bot, code_dim) * 0.05)
+        # EMA state para top.
+        self.register_buffer('ema_cluster_top', torch.zeros(num_codes_top))
+        self.register_buffer('ema_w_top', torch.zeros(num_codes_top, code_dim))
+        # EMA state para bottom.
+        self.register_buffer('ema_cluster_bot', torch.zeros(num_codes_bot))
+        self.register_buffer('ema_w_bot', torch.zeros(num_codes_bot, code_dim))
+
+        # v12.8: Normalização opcional dentro do VQ (substitui LayerNorm v12.7 que degradou).
+        # - "none": sem normalização (preserva estatística original — RECOMENDADO para VQ-VAE)
+        # - "weightnorm": weight normalization (EnCodec-style, preserva magnitude)
+        # - "rmsnorm": RMSNorm (preserva magnitude, mais barato que LayerNorm)
+        # Nota: LayerNorm v12.7 foi REMOVIDO pois degradou resultados (17× pior loss).
+        if norm_type == "rmsnorm":
+            # RMSNorm: y = x / sqrt(mean(x²) + eps) * gamma
+            self.rms_pre_top = RMSNorm(code_dim)
+            self.rms_pre_bot = RMSNorm(code_dim)
+            self.rms_post_combine = RMSNorm(code_dim)
+        else:
+            # "none" ou "weightnorm" (weightnorm é aplicado nos encoders, não aqui).
+            self.rms_pre_top = nn.Identity()
+            self.rms_pre_bot = nn.Identity()
+            self.rms_post_combine = nn.Identity()
+
+        # Contador de épocas.
+        self.register_buffer('epoch_counter', torch.zeros(1, dtype=torch.long))
+
+        # Estatísticas de uso.
+        self.register_buffer('usage_top', torch.zeros(num_codes_top))
+        self.register_buffer('usage_bot', torch.zeros(num_codes_bot))
+
+        # Temperatura Goose atual (atualizada via update_temperature).
+        self.register_buffer('current_temp', torch.tensor(goose_temp_init))
+
+    def update_temperature(self, grad_recon: float = None, grad_vq: float = None):
+        """Atualiza temperatura Goose baseada na época atual (schedule).
+        
+        v13.9: Se grad_recon e grad_vq fornecidos, usa mas_weight adaptativo:
+            τ* = ||∇L_recon|| / ||∇L_vq||
+        Caso contrário, usa cosine decay padrão.
+        """
+        epoch = int(self.epoch_counter.item())
+        progress = min(epoch / max(self.total_epochs, 1), 1.0)
+        
+        if grad_recon is not None and grad_vq is not None and grad_vq > 1e-8:
+            # v13.9: τ* adaptativo via mas_weight formula.
+            from .mathematical_optimizations import compute_goose_tau_adaptive
+            prev_tau = float(self.current_temp)
+            temp = compute_goose_tau_adaptive(grad_recon, grad_vq, prev_tau,
+                                              tau_min=self.goose_temp_final,
+                                              tau_max=self.goose_temp_init)
+        elif self.goose_schedule == "cosine":
+            temp = self.goose_temp_final + 0.5 * (self.goose_temp_init - self.goose_temp_final) * \
+                   (1 + math.cos(math.pi * progress))
+        else:  # linear
+            temp = self.goose_temp_init + (self.goose_temp_final - self.goose_temp_init) * progress
+        self.current_temp.fill_(temp)
+        return float(temp)
+
+    def _quantize_level(self, z: torch.Tensor, codebook: torch.Tensor,
+                         ema_cluster: torch.Tensor, ema_w: torch.Tensor,
+                         use_goose: bool = True) -> Tuple[torch.Tensor, torch.Tensor, int]:
+        """Quantiza um nível (top ou bottom) com EMA + Goose opcional."""
+        B, D = z.shape
+        # Distâncias.
+        dist = (
+            z.pow(2).sum(dim=1, keepdim=True)
+            + codebook.pow(2).sum(dim=1)
+            - 2 * z @ codebook.t()
+        )  # (B, K)
+
+        if use_goose and self.training:
+            # Goose VQ: Gumbel-softmax sampling.
+            logits = -dist / max(float(self.current_temp), 1e-6)
+            codes_soft = F.gumbel_softmax(logits, tau=float(self.current_temp), hard=True)
+            codes = codes_soft.argmax(dim=1)
+            z_q = codes_soft @ codebook
+        else:
+            # argmin (eval ou fallback).
+            codes = dist.argmin(dim=1)
+            z_q = codebook[codes]
+
+        # EMA update (apenas em treino).
+        n_restarted = 0
+        if self.training:
+            one_hot = F.one_hot(codes, codebook.size(0)).float()
+            new_cluster = one_hot.sum(dim=0)
+            # v13.3fix: Save OLD values before EMA update for cb_update computation
+            old_cluster = ema_cluster.clone()
+            old_w = ema_w.clone()
+            ema_cluster.mul_(self.ema_decay).add_(new_cluster, alpha=1 - self.ema_decay)
+            new_w = one_hot.t() @ z
+            ema_w.mul_(self.ema_decay).add_(new_w, alpha=1 - self.ema_decay)
+            # v13.3: Correções matemáticas no EMA (mal-conditioning do codebook).
+            # Use OLD (pre-update) values for cb_update, as required by proper EMA formula.
+            n_old = old_cluster.sum()
+            # 1. Smoothing do cluster size (reduz condition number).
+            cluster_mean = old_cluster.mean()
+            cluster_smoothed = 0.9 * old_cluster + 0.1 * cluster_mean
+            # 2. Clipping logarítmico do cluster size (comprime escala dinâmica).
+            eps_cb = 1e-5
+            cluster_clipped = torch.log1p(cluster_smoothed / eps_cb) / \
+                torch.log1p(cluster_smoothed.max() / eps_cb + 1e-12)
+            # 3. Renormalizar para magnitude original.
+            cluster_final = cluster_clipped * n_old / (cluster_clipped.sum() + 1e-12)
+            smoothed = (cluster_final + eps_cb) / (n_old + codebook.size(0) * eps_cb) * n_old
+            # v13.3: Clamp do codebook update para prevenir NaN/Inf.
+            # Use OLD ema_w (pre-update) for cb_update
+            cb_update = old_w / smoothed.unsqueeze(1)
+            cb_update = torch.clamp(cb_update, -1e4, 1e4)
+            codebook.copy_(cb_update)
+
+            # Dead code restart.
+            if (int(self.epoch_counter.item()) + 1) % self.dead_code_restart_every == 0:
+                dead_mask = ema_cluster < self.dead_code_threshold
+                n_dead = int(dead_mask.sum().item())
+                if n_dead > 0:
+                    rand_idx = torch.randint(0, B, (n_dead,), device=z.device)
+                    replacement = z[rand_idx]
+                    codebook.data[dead_mask] = replacement.detach()
+                    ema_cluster[dead_mask] = 1.0
+                    ema_w[dead_mask] = replacement.detach()
+                    n_restarted = n_dead
+                    # v13.1: limpeza leve após dead code restart (libera tensores temporários).
+                    import gc as _gc
+                    _gc.collect(0)
+
+        return z_q, codes, n_restarted
+
+    def forward(self, z_e: torch.Tensor) -> Dict[str, torch.Tensor]:
+        """
+        z_e: (B, D) — saída contínua do encoder.
+
+        Retorna dict com:
+            z_q_top: (B, D) — vetores quantizados do nível top
+            z_q_bot: (B, D) — vetores quantizados do nível bottom (residual)
+            z_q_combined: (B, D) — z_q_top + z_q_bot (soma)
+            codes_top: (B,)
+            codes_bot: (B,)
+            loss: escalar (commitment top + bottom)
+            stats: dict com estatísticas
+        """
+        # v12.8: RMSNorm pré-top (preserva magnitude, mais estável que LayerNorm).
+        z_e_norm = self.rms_pre_top(z_e)
+
+        # Nível top: quantiza z_e (normalizado) diretamente.
+        z_q_top, codes_top, n_restart_top = self._quantize_level(
+            z_e_norm, self.codebook_top, self.ema_cluster_top, self.ema_w_top,
+            use_goose=True,
+        )
+
+        # Nível bottom: quantiza o RESÍDUO z_e - z_q_top.detach() (com RMSNorm).
+        residual = z_e - z_q_top.detach()
+        residual_norm = self.rms_pre_bot(residual)
+        z_q_bot, codes_bot, n_restart_bot = self._quantize_level(
+            residual_norm, self.codebook_bot, self.ema_cluster_bot, self.ema_w_bot,
+            use_goose=True,
+        )
+
+        # Commitment losses (computadas com z_e original, não normalizado).
+        commit_top = F.mse_loss(z_e, z_q_top.detach())
+        commit_bot = F.mse_loss(residual, z_q_bot.detach())
+        loss = self.beta * (commit_top + commit_bot)
+
+        # Straight-through.
+        z_q_top_st = z_e + (z_q_top - z_e).detach()
+        z_q_bot_st = residual + (z_q_bot - residual).detach()
+        z_q_combined = z_q_top_st + z_q_bot_st
+        # v12.8: RMSNorm pós-combinação (preserva magnitude, mais estável que LayerNorm).
+        z_q_combined = self.rms_post_combine(z_q_combined)
+
+        # Estatísticas.
+        with torch.no_grad():
+            self.usage_top.add_(torch.bincount(codes_top, minlength=self.num_codes_top).float())
+            self.usage_bot.add_(torch.bincount(codes_bot, minlength=self.num_codes_bot).float())
+            n_used_top = int((self.usage_top > 0).sum().item())
+            n_used_bot = int((self.usage_bot > 0).sum().item())
+
+            def _cb_ppl(codes, K):
+                if len(codes) == 0:
+                    return 0.0
+                from collections import Counter
+                counter = Counter(codes.tolist())
+                probs = np.array([c / len(codes) for c in counter.values()])
+                entropy = -np.sum(probs * np.log(probs + 1e-12))
+                return float(math.exp(entropy)) if entropy > 0 else 0.0
+
+            cb_ppl_top = _cb_ppl(codes_top, self.num_codes_top)
+            cb_ppl_bot = _cb_ppl(codes_bot, self.num_codes_bot)
+
+        stats = {
+            'n_used_top': n_used_top,
+            'n_used_bot': n_used_bot,
+            'n_dead_top': self.num_codes_top - n_used_top,
+            'n_dead_bot': self.num_codes_bot - n_used_bot,
+            'usage_ratio_top': n_used_top / self.num_codes_top,
+            'usage_ratio_bot': n_used_bot / self.num_codes_bot,
+            'codebook_ppl_top': cb_ppl_top,
+            'codebook_ppl_bot': cb_ppl_bot,
+            'n_restarted_top': n_restart_top,
+            'n_restarted_bot': n_restart_bot,
+            'goose_temp': float(self.current_temp),
+        }
+
+        return {
+            'z_q_top': z_q_top_st,
+            'z_q_bot': z_q_bot_st,
+            'z_q_combined': z_q_combined,
+            'codes_top': codes_top,
+            'codes_bot': codes_bot,
+            'loss': loss,
+            'stats': stats,
+        }
+
+    def increment_epoch(self):
+        self.epoch_counter += 1
+        self.update_temperature()
+
+    def reset_usage_stats(self):
+        self.usage_top.zero_()
+        self.usage_bot.zero_()
+
+
+class HierarchicalVQVAE2(nn.Module):
+    """VQ-VAE-2 completo com encoder/decoder hierárquico para múltiplas modalidades.
+
+    Arquitetura:
+        x → [Encoder shared] → z_e
+                              → [Top VQ]  → z_q_top  (estrutura global)
+                              → [Bot VQ]  → z_q_bot  (detalhes residuais)
+                              → [Decoder shared] → x_recon
+    """
+
+    def __init__(
+        self,
+        modalities: Dict[str, int],
+        code_dim: int = 32,
+        num_codes_top: int = 512,
+        num_codes_bot: int = 512,
+        hidden: int = 128,
+        beta: float = 0.25,
+        ema_decay: float = 0.99,
+        dead_code_threshold: float = 1.0,
+        dead_code_restart_every: int = 3,
+        goose_temp_init: float = 2.0,
+        goose_temp_final: float = 0.5,
+        goose_schedule: str = "cosine",
+        total_epochs: int = 25,
+        norm_type: str = "none",  # v12.8: "none" | "weightnorm" | "rmsnorm"
+        rmsnorm_in_vq: bool = False,  # v12.8: RMSNorm dentro do VQ (default False)
+    ):
+        super().__init__()
+        self.modality_names = list(modalities.keys())
+        self.modality_dims = modalities
+        self.code_dim = code_dim
+        self.norm_type = norm_type  # v12.8
+
+        # v12.8: Encoders e decoders por modalidade.
+        # - "none": LayerNorm interno dos MLPs (default, melhor validado em v12.7)
+        # - "weightnorm": aplica weight_norm nos Linear (EnCodec-style)
+        # - "rmsnorm": usa RMSNorm em vez de LayerNorm entre camadas MLP
+        def make_encoder(dim, hidden, code_dim, norm_type):
+            if norm_type == "weightnorm":
+                layers = [
+                    nn.utils.weight_norm(nn.Linear(dim, hidden)), nn.GELU(),
+                    nn.utils.weight_norm(nn.Linear(hidden, hidden // 2)), nn.GELU(),
+                    nn.utils.weight_norm(nn.Linear(hidden // 2, code_dim)),
+                ]
+            elif norm_type == "rmsnorm":
+                layers = [
+                    nn.Linear(dim, hidden), RMSNorm(hidden), nn.GELU(),
+                    nn.Linear(hidden, hidden // 2), RMSNorm(hidden // 2), nn.GELU(),
+                    nn.Linear(hidden // 2, code_dim),
+                ]
+            else:  # "none" — usa LayerNorm interno (default, melhor validado)
+                layers = [
+                    nn.Linear(dim, hidden), nn.LayerNorm(hidden), nn.GELU(),
+                    nn.Linear(hidden, hidden // 2), nn.LayerNorm(hidden // 2), nn.GELU(),
+                    nn.Linear(hidden // 2, code_dim),
+                ]
+            return nn.Sequential(*layers)
+
+        def make_decoder(code_dim, hidden, dim, norm_type):
+            if norm_type == "weightnorm":
+                layers = [
+                    nn.utils.weight_norm(nn.Linear(code_dim, hidden // 2)), nn.GELU(),
+                    nn.utils.weight_norm(nn.Linear(hidden // 2, hidden)), nn.GELU(),
+                    nn.utils.weight_norm(nn.Linear(hidden, dim)),
+                ]
+            elif norm_type == "rmsnorm":
+                layers = [
+                    nn.Linear(code_dim, hidden // 2), RMSNorm(hidden // 2), nn.GELU(),
+                    nn.Linear(hidden // 2, hidden), RMSNorm(hidden), nn.GELU(),
+                    nn.Linear(hidden, dim),
+                ]
+            else:  # "none"
+                layers = [
+                    nn.Linear(code_dim, hidden // 2), nn.LayerNorm(hidden // 2), nn.GELU(),
+                    nn.Linear(hidden // 2, hidden), nn.LayerNorm(hidden), nn.GELU(),
+                    nn.Linear(hidden, dim),
+                ]
+            return nn.Sequential(*layers)
+
+        self.encoders = nn.ModuleDict({
+            name: make_encoder(dim, hidden, code_dim, norm_type)
+            for name, dim in modalities.items()
+        })
+        self.decoders = nn.ModuleDict({
+            name: make_decoder(code_dim, hidden, dim, norm_type)
+            for name, dim in modalities.items()
+        })
+
+        # VQ hierárquico compartilhado entre todas as modalidades.
+        # v12.8: RMSNorm opcional dentro do VQ (substitui LayerNorm v12.7 que degradou).
+        vq_norm_type = "rmsnorm" if (rmsnorm_in_vq and norm_type == "rmsnorm") else "none"
+        self.vq = HierarchicalVectorQuantizer(
+            num_codes_top=num_codes_top,
+            num_codes_bot=num_codes_bot,
+            code_dim=code_dim,
+            beta=beta,
+            ema_decay=ema_decay,
+            dead_code_threshold=dead_code_threshold,
+            dead_code_restart_every=dead_code_restart_every,
+            goose_temp_init=goose_temp_init,
+            goose_temp_final=goose_temp_final,
+            goose_schedule=goose_schedule,
+            total_epochs=total_epochs,
+            norm_type=vq_norm_type,
+        )
+
+    def forward(self, batch: Dict[str, torch.Tensor]) -> Dict[str, torch.Tensor]:
+        """batch: {modality_name: (B, input_dim)}."""
+        out = {'reconstructions': {}, 'codes_top': {}, 'codes_bot': {}, 'vq_loss': 0.0,
+               'recon_loss': 0.0, 'stats': {}}
+        total_vq = 0.0
+        total_recon = 0.0
+        n = 0
+        all_codes_top = []
+        all_codes_bot = []
+        vq_out_first_stats = None  # Capture stats from first VQ forward
+        for name, x in batch.items():
+            if name not in self.encoders:
+                continue
+            # v13.3: Sanitização de NaN/Inf no input (resolve bug identificado no teste adversarial).
+            x = torch.nan_to_num(x, nan=0.0, posinf=1e4, neginf=-1e4)
+            # v12.8: forward limpo (sem LayerNorms extras v12.7 que degradaram resultados).
+            # Encoder (com normalização interna configurável via norm_type).
+            z_e = self.encoders[name](x)
+            # v13.3: Sanitização após encoder (pode gerar NaN com inputs extremos).
+            z_e = torch.nan_to_num(z_e, nan=0.0, posinf=1e4, neginf=-1e4)
+            # VQ hierárquico (com RMSNorm opcional via rmsnorm_in_vq).
+            vq_out = self.vq(z_e)
+            # Capture stats from the first modality's VQ forward (no redundant second pass)
+            if vq_out_first_stats is None:
+                vq_out_first_stats = vq_out['stats']
+            z_q = vq_out['z_q_combined']
+            # v13.3: Sanitização após VQ (quantização pode produzir valores extremos).
+            z_q = torch.nan_to_num(z_q, nan=0.0, posinf=1e4, neginf=-1e4)
+            # Decoder (com normalização interna configurável via norm_type).
+            x_recon = self.decoders[name](z_q)
+            # v13.3: Sanitização final da reconstrução.
+            x_recon = torch.nan_to_num(x_recon, nan=0.0, posinf=1e4, neginf=-1e4)
+            recon_loss = F.mse_loss(x_recon, x)
+
+            out['reconstructions'][name] = x_recon
+            out['codes_top'][name] = vq_out['codes_top']
+            out['codes_bot'][name] = vq_out['codes_bot']
+            all_codes_top.append(vq_out['codes_top'])
+            all_codes_bot.append(vq_out['codes_bot'])
+            total_vq += vq_out['loss']
+            total_recon += recon_loss
+            n += 1
+
+        out['vq_loss'] = total_vq / max(n, 1)
+        out['recon_loss'] = total_recon / max(n, 1)
+        out['total_loss'] = out['vq_loss'] + out['recon_loss']
+        # Use stats already computed in the main loop — avoid redundant VQ forward
+        # that would sample different random codes and give inconsistent stats.
+        if vq_out_first_stats is not None:
+            out['stats'] = vq_out_first_stats
+        return out
+
+    def cross_modal_translate(
+        self, x: torch.Tensor, src: str, tgt: str
+    ) -> torch.Tensor:
+        """Traduz de uma modalidade para outra via codebook compartilhado."""
+        # v12.8: forward limpo (sem LayerNorms extras v12.7).
+        z_e = self.encoders[src](x)
+        vq_out = self.vq(z_e)
+        z_q = vq_out['z_q_combined']
+        return self.decoders[tgt](z_q)

xeon_runtime.py

@@ -0,0 +1,581 @@
+"""xeon_runtime.py — Intel Xeon runtime (V6): AVX512 + AMX_INT8 + IPEX + OneDNN + FP16.
+
+═══════════════════════════════════════════════════════════════════════════════
+V6 UPGRADE
+═══════════════════════════════════════════════════════════════════════════════
+
+V5 only set OMP/MKL threads + KMP_AFFINITY. V6 adds:
+
+1. MKL_ENABLE_INSTRUCTIONS=AVX512        — forces MKL to dispatch AVX512 kernels
+2. ONEDNN_MAX_CPU_ISA=AMX_INT8           — lets oneDNN use AMX INT8 tiles
+3. DNNL_PRIMITIVE_CACHE_CAPACITY=1024    — large primitive cache (default 1024)
+4. MKL_DYNAMIC=FALSE                     — disables MKL dynamic thread adjustment
+5. IPEX (intel_extension_for_pytorch)    — Intel PyTorch extension
+   - ipex.optimize(model) on the BiGRU_T model
+   - torch.cpu.amp.autocast(dtype=torch.float16) for FP16 inference
+6. FP16 benchmark: 8000×8000 matmul, TFLOPS measurement
+7. libvirt AMX activation helper — exposes amx-tile/amx-int8/amx-bf16 to a VM
+   via host-passthrough CPU mode + feature policy='require'
+
+The runtime is **always activated** (per user request: "sempre ativar otimização
+para Xeon AVX512"). It degrades gracefully if IPEX / libvirt / AMX are not
+available, but it never silently skips the optimization step.
+
+═══════════════════════════════════════════════════════════════════════════════
+USAGE
+═══════════════════════════════════════════════════════════════════════════════
+
+    from bigru_t.utils.xeon_runtime import optimize_xeon_environment
+    N_CORES = optimize_xeon_environment()           # call ONCE, before torch
+    # ... safe to import torch, ipex, etc. ...
+
+For VM AMX exposure (requires libvirt-python and root):
+    from bigru_t.utils.xeon_runtime import ativar_amx_na_vm
+    ativar_amx_na_vm("my_vm_name")
+
+═══════════════════════════════════════════════════════════════════════════════
+"""
+from __future__ import annotations
+import os
+import sys
+import time
+import logging
+import platform
+from typing import Optional, Tuple, Dict, Any
+
+logger = logging.getLogger(__name__)
+
+_NUCLEOS_ALOCADOS: Optional[int] = None
+_IPEX_AVAILABLE: Optional[bool] = None
+_AMX_CAPABLE: Optional[bool] = None
+_V6_INIT_DONE: bool = False
+
+
+# ============================================================================
+# Helpers
+# ============================================================================
+
+def _detect_physical_cores() -> int:
+    """Detect physical cores actually available to this process.
+
+    Respects cgroup limits without requiring root. Falls back to logical
+    cpu count if psutil is unavailable.
+    """
+    try:
+        import psutil
+        n_phys = psutil.cpu_count(logical=False) or 1
+    except ImportError:
+        try:
+            with open("/proc/cpuinfo", "r") as f:
+                cores = set()
+                for line in f:
+                    if line.startswith("core id"):
+                        cores.add(line.strip())
+                n_phys = len(cores) or 1
+        except OSError:
+            n_phys = 1
+
+    try:
+        n_affine = len(os.sched_getaffinity(0))
+        n_logical = os.cpu_count() or 1
+        if n_affine < n_logical:
+            n_phys = max(1, n_affine // 2)
+        else:
+            n_phys = min(n_phys, n_affine)
+    except (AttributeError, OSError):
+        pass
+
+    return max(1, n_phys)
+
+
+def _read_cpu_flags() -> str:
+    try:
+        with open("/proc/cpuinfo", "r") as f:
+            for line in f:
+                if line.startswith("flags"):
+                    return line
+    except OSError:
+        pass
+    return ""
+
+
+def get_avx512_capability() -> Tuple[bool, str]:
+    """Check if the host CPU supports AVX512 VNNI."""
+    flags = _read_cpu_flags()
+    if "avx512_vnni" in flags:
+        return True, "AVX512_VNNI (full INT8 acceleration)"
+    elif "avx512f" in flags:
+        return True, "AVX512F (no VNNI; INT8 falls back to AVX512F)"
+    elif "avx2" in flags:
+        return False, "AVX2 only (INT8 quantization works but slower)"
+    else:
+        return False, "Legacy SSE (INT8 quantization not recommended)"
+
+
+def get_amx_capability() -> Tuple[bool, str]:
+    """V6: check if AMX (Advanced Matrix Extensions) is available."""
+    global _AMX_CAPABLE
+    flags = _read_cpu_flags()
+    has_tile = "amx_tile" in flags
+    has_int8 = "amx_int8" in flags
+    has_bf16 = "amx_bf16" in flags
+    if has_tile and has_int8 and has_bf16:
+        _AMX_CAPABLE = True
+        return True, "AMX (tile + int8 + bf16) — full AMX acceleration"
+    elif has_tile:
+        _AMX_CAPABLE = True
+        return True, f"AMX tile only (int8={has_int8}, bf16={has_bf16})"
+    else:
+        _AMX_CAPABLE = False
+        return False, "AMX not available (AVX512 path will be used)"
+
+
+def _try_import_ipex() -> Optional[Any]:
+    """V6: try to import IPEX (Intel Extension for PyTorch).
+
+    Returns the ipex module if available, else None. Caches the result.
+    """
+    global _IPEX_AVAILABLE
+    if _IPEX_AVAILABLE is False:
+        return None
+    try:
+        import intel_extension_for_pytorch as ipex  # type: ignore
+        _IPEX_AVAILABLE = True
+        return ipex
+    except (ImportError, AttributeError, OSError) as e:
+        _IPEX_AVAILABLE = False
+        if _V6_INIT_DONE is False:
+            logger.info(f"[Xeon V6] IPEX não disponível: {type(e).__name__}: {e}")
+            logger.info("[Xeon V6] Continuando com OneDNN/MKL nativo do PyTorch.")
+        return None
+
+
+# ============================================================================
+# FP16 benchmark (V6 — user-provided code)
+# ============================================================================
+
+def benchmark_fp16_matmul(size: int = 8000, warmup: int = 1, iters: int = 3) -> Dict[str, float]:
+    """V6: FP16 matmul benchmark for Xeon AVX512/AMX.
+
+    Runs `size`×`size` FP16 matmul `iters` times and reports:
+        - best_time_ms: lowest wall time
+        - best_tflops: best achieved TFLOPS
+        - avg_tflops: average TFLOPS
+
+    Returns empty dict if torch unavailable.
+    """
+    try:
+        import torch
+    except ImportError:
+        return {}
+
+    results: Dict[str, float] = {}
+    try:
+        # Warmup
+        a = torch.randn(size, size, dtype=torch.float16)
+        b = torch.randn(size, size, dtype=torch.float16)
+        for _ in range(warmup):
+            _ = torch.matmul(a, b)
+        # Bench
+        times = []
+        for _ in range(iters):
+            t0 = time.perf_counter()
+            _ = torch.matmul(a, b)
+            times.append(time.perf_counter() - t0)
+        best_t = min(times)
+        avg_t = sum(times) / len(times)
+        # 2*size^3 FLOPs per matmul (M*N*K)
+        flops = 2.0 * (size ** 3)
+        results["best_time_ms"] = best_t * 1000.0
+        results["avg_time_ms"] = avg_t * 1000.0
+        results["best_tflops"] = flops / best_t / 1e12
+        results["avg_tflops"] = flops / avg_t / 1e12
+        results["matrix_size"] = float(size)
+    except (RuntimeError, MemoryError) as e:
+        logger.warning(f"[Xeon V6] FP16 benchmark failed: {e}")
+        results["error"] = str(e)
+    return results
+
+
+def benchmark_int8_matmul(size: int = 4096, warmup: int = 1, iters: int = 3) -> Dict[str, float]:
+    """V6: INT8 matmul benchmark — uses AMX_INT8 when available via oneDNN.
+
+    Falls back to FP32 if INT8 path is unavailable.
+    """
+    try:
+        import torch
+    except ImportError:
+        return {}
+
+    results: Dict[str, float] = {}
+    try:
+        a = torch.randint(-127, 127, (size, size), dtype=torch.int8)
+        b = torch.randint(-127, 127, (size, size), dtype=torch.int8)
+        # Warmup
+        for _ in range(warmup):
+            _ = torch.matmul(a.float(), b.float())
+        times = []
+        for _ in range(iters):
+            t0 = time.perf_counter()
+            _ = torch.matmul(a.float(), b.float())
+            times.append(time.perf_counter() - t0)
+        best_t = min(times)
+        flops = 2.0 * (size ** 3)
+        results["int8_best_time_ms"] = best_t * 1000.0
+        results["int8_best_tflops"] = flops / best_t / 1e12
+        results["int8_matrix_size"] = float(size)
+    except (RuntimeError, MemoryError) as e:
+        results["int8_error"] = str(e)
+    return results
+
+
+# ============================================================================
+# Main entry point: optimize_xeon_environment (V6)
+# ============================================================================
+
+def optimize_xeon_environment(
+    verbose: bool = True,
+    force_ipex: bool = False,
+) -> int:
+    """V6: configure Intel Xeon AVX512 + AMX_INT8 + IPEX + OneDNN.
+
+    Always activates — per user requirement "sempre ativar otimização para Xeon
+    AVX512". Idempotent. Sets every environment variable that influences MKL,
+    OpenMP, oneDNN, and (optionally) IPEX runtime behavior.
+
+    Args:
+        verbose: print configuration summary to stdout
+        force_ipex: if True, raise when IPEX import fails. Default False —
+            degrade gracefully to native PyTorch oneDNN.
+
+    Returns:
+        Number of physical cores allocated to this process.
+    """
+    global _NUCLEOS_ALOCADOS, _V6_INIT_DONE
+
+    if _NUCLEOS_ALOCADOS is not None and _V6_INIT_DONE:
+        return _NUCLEOS_ALOCADOS
+
+    n_phys = _detect_physical_cores()
+
+    # ───────────────────────────────────────────────────────────────────────
+    # V6 — Environment variables (user-provided block)
+    # ───────────────────────────────────────────────────────────────────────
+    os.environ["MKL_ENABLE_INSTRUCTIONS"] = "AVX512"
+    NUM_CORES = str(n_phys)
+    os.environ["MKL_NUM_THREADS"] = NUM_CORES
+    os.environ["OMP_NUM_THREADS"] = NUM_CORES
+    os.environ["MKL_DYNAMIC"] = "FALSE"
+    os.environ["DNNL_PRIMITIVE_CACHE_CAPACITY"] = "1024"
+    os.environ["ONEDNN_MAX_CPU_ISA"] = "AMX_INT8"
+
+    # ───────────────────────────────────────────────────────────────────────
+    # V5 (kept) — OpenMP thread pinning
+    # ───────────────────────────────────────────────────────────────────────
+    os.environ.setdefault("KMP_AFFINITY", "granularity=fine,compact,1,0")
+    os.environ.setdefault("KMP_BLOCKTIME", "1")
+    os.environ.setdefault("TOKENIZERS_PARALLELISM", "true")
+
+    # ───────────────────────────────────────────────────────────────────────
+    # PyTorch backend flags
+    # ───────────────────────────────────────────────────────────────────────
+    try:
+        import torch
+        torch.set_num_threads(n_phys)
+        try:
+            torch.set_num_interop_threads(1)
+        except RuntimeError:
+            # V6.5: já inicializado (e.g., bigru_t package importou torch antes).
+            # Silenciosamente ignora — o paralelismo já está configurado.
+            pass
+        if hasattr(torch.backends, "mkldnn"):
+            torch.backends.mkldnn.enabled = True
+        if hasattr(torch.backends, "quantized"):
+            try:
+                torch.backends.quantized.engine = "fbgemm"
+            except (RuntimeError, AttributeError):
+                pass
+        # V6: enable TF32 for Ampere+ / Sapphire Rapids (irrelevant on CPU but
+        # harmless) and ensure oneDNN verbose is silent.
+        try:
+            torch.backends.cuda.matmul.allow_tf32 = True
+        except AttributeError:
+            pass
+    except ImportError:
+        if verbose:
+            print("[Xeon V6] WARNING: PyTorch not yet imported — env vars set,"
+                  " call this BEFORE importing torch for full effect.")
+
+    # ───────────────────────────────────────────────────────────────────────
+    # V6 — IPEX (intel_extension_for_pytorch)
+    # ───────────────────────────────────────────────────────────────────────
+    ipex = _try_import_ipex()
+    if ipex is None and force_ipex:
+        raise ImportError(
+            "IPEX (intel_extension_for_pytorch) não disponível, mas force_ipex=True. "
+            "Instale com: pip install intel-extension-for-pytorch"
+        )
+
+    # ───────────────────────────────────────────────────────────────────────
+    # V6 — AMX capability check
+    # ───────────────────────────────────────────────────────────────────────
+    amx_ok, amx_desc = get_amx_capability()
+    avx_ok, avx_desc = get_avx512_capability()
+
+    _NUCLEOS_ALOCADOS = n_phys
+    _V6_INIT_DONE = True
+
+    if verbose:
+        print("\n" + "=" * 72)
+        print(f"[Xeon Runtime V6] Intel Xeon optimization activated")
+        print("=" * 72)
+        print(f"  Physical cores        : {n_phys}")
+        print(f"  MKL_NUM_THREADS       : {os.environ['MKL_NUM_THREADS']}")
+        print(f"  OMP_NUM_THREADS       : {os.environ['OMP_NUM_THREADS']}")
+        print(f"  MKL_DYNAMIC           : {os.environ['MKL_DYNAMIC']}")
+        print(f"  MKL_ENABLE_INSTRUCTIONS: {os.environ['MKL_ENABLE_INSTRUCTIONS']}")
+        print(f"  ONEDNN_MAX_CPU_ISA    : {os.environ['ONEDNN_MAX_CPU_ISA']}")
+        print(f"  DNNL_PRIMITIVE_CACHE  : {os.environ['DNNL_PRIMITIVE_CACHE_CAPACITY']}")
+        print(f"  KMP_AFFINITY          : {os.environ['KMP_AFFINITY']}")
+        print(f"  KMP_BLOCKTIME         : {os.environ['KMP_BLOCKTIME']}")
+        print(f"  AVX512                : {avx_desc}")
+        print(f"  AMX                   : {amx_desc}")
+        print(f"  IPEX                  : "
+              f"{'available' if ipex is not None else 'not installed (using native oneDNN)'}")
+        print("=" * 72 + "\n")
+
+    return n_phys
+
+
+# ============================================================================
+# V6 — apply_ipex_optimization (model-level)
+# ============================================================================
+
+def apply_ipex_optimization(model, dtype=None, optimizer=None):
+    """V6: apply ipex.optimize() to a model.
+
+    Returns (model, optimizer) tuple. If IPEX is not available, returns the
+    inputs unchanged. The model is modified in-place when IPEX is present.
+
+    Args:
+        model: torch.nn.Module
+        dtype: optional torch.dtype for the model (e.g. torch.bfloat16)
+        optimizer: optional torch.optim.Optimizer to also optimize
+    """
+    ipex = _try_import_ipex()
+    if ipex is None:
+        return model, optimizer
+    try:
+        import torch
+        if dtype is not None:
+            model = model.to(dtype)
+        if optimizer is not None:
+            model, optimizer = ipex.optimize(model=model, optimizer=optimizer, dtype=dtype)
+        else:
+            model = ipex.optimize(model=model, dtype=dtype)
+        logger.info(f"[Xeon V6] ipex.optimize applied (dtype={dtype})")
+    except (RuntimeError, AttributeError, TypeError) as e:
+        logger.warning(f"[Xeon V6] ipex.optimize failed: {e}")
+    return model, optimizer
+
+
+# ============================================================================
+# V6 — FP16 autocast context manager
+# ============================================================================
+
+class fp16_autocast:
+    """V6: FP16 CPU autocast context manager.
+
+    Uses torch.cpu.amp.autocast(dtype=torch.float16) when available. Falls
+    back to a no-op context manager if autocast is not supported.
+
+    Usage:
+        with fp16_autocast():
+            y = model(x)
+    """
+
+    def __init__(self, enabled: bool = True):
+        self.enabled = enabled
+        self._ctx = None
+
+    def __enter__(self):
+        if not self.enabled:
+            return self
+        try:
+            import torch
+            if hasattr(torch.cpu, "amp") and hasattr(torch.cpu.amp, "autocast"):
+                self._ctx = torch.cpu.amp.autocast(dtype=torch.float16)
+                self._ctx.__enter__()
+        except (ImportError, RuntimeError, AttributeError):
+            self._ctx = None
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        if self._ctx is not None:
+            self._ctx.__exit__(exc_type, exc_val, exc_tb)
+            self._ctx = None
+        return False
+
+
+# ============================================================================
+# V6 — libvirt AMX activation helper (user-provided code, integrated)
+# ============================================================================
+
+def ativar_amx_na_vm(nome_vm: str, qemu_uri: str = "qemu:///system") -> bool:
+    """V6: ativa AMX (amx-tile, amx-int8, amx-bf16) numa VM via libvirt.
+
+    Requer:
+        - pip install libvirt-python
+        - libvirtd rodando localmente (qemu:///system)
+        - permissão de root (ou membro do grupo libvirt)
+        - CPU física com AMX (verificado por get_amx_capability())
+
+    Implementação:
+        1. Conecta ao daemon libvirt
+        2. Busca a VM pelo nome
+        3. Lê o XML persistente (flag VIR_DOMAIN_XML_INACTIVE = 2)
+        4. Garante <cpu mode='host-passthrough'>
+        5. Adiciona <feature policy='require' name='amx-tile|int8|bf16'/>
+        6. Reescreve o XML via defineXML()
+
+    Retorna True se a VM foi atualizada com sucesso, False caso contrário.
+    Requer reboot da VM para aplicar.
+
+    Args:
+        nome_vm: nome da máquina virtual no libvirt
+        qemu_uri: URI do libvirt (default: qemu:///system)
+
+    Raises:
+        ImportError: se libvirt-python não estiver instalado
+        RuntimeError: se a conexão com libvirt falhar
+    """
+    try:
+        import libvirt  # type: ignore
+    except ImportError as e:
+        raise ImportError(
+            "libvirt-python não instalado. Rode: pip install libvirt-python "
+            "(também requer libvirt-dev no sistema: apt install libvirt-dev)"
+        ) from e
+
+    try:
+        import xml.etree.ElementTree as ET
+    except ImportError:
+        return False
+
+    # Pré-checa AMX no host físico
+    amx_ok, amx_desc = get_amx_capability()
+    if not amx_ok:
+        print(f"[AMX VM] Host físico não tem AMX ({amx_desc}).")
+        print("         Não adianta ativar AMX na VM — o host precisa suportar.")
+        return False
+
+    try:
+        conn = libvirt.open(qemu_uri)
+        if conn is None:
+            print(f"[AMX VM] Falha ao abrir conexão com {qemu_uri}")
+            return False
+
+        try:
+            dom = conn.lookupByName(nome_vm)
+        except libvirt.libvirtError:
+            print(f"[AMX VM] VM '{nome_vm}' não encontrada.")
+            conn.close()
+            return False
+
+        xml_atual = dom.XMLDesc(2)  # VIR_DOMAIN_XML_INACTIVE
+        root = ET.fromstring(xml_atual)
+        cpu_elem = root.find("cpu")
+
+        if cpu_elem is None:
+            cpu_elem = ET.SubElement(root, "cpu", mode="host-passthrough")
+            print("[AMX VM] Tag <cpu> criada com mode='host-passthrough'.")
+        else:
+            cpu_elem.set("mode", "host-passthrough")
+            print("[AMX VM] CPU mode atualizado para host-passthrough.")
+
+        flags_para_adicionar = ["amx-tile", "amx-int8", "amx-bf16"]
+        added = []
+        for flag in flags_para_adicionar:
+            existing = cpu_elem.find(f"./feature[@name='{flag}']")
+            if existing is None:
+                ET.SubElement(cpu_elem, "feature", policy="require", name=flag)
+                added.append(flag)
+                print(f"[AMX VM] Feature adicionada: {flag}")
+
+        novo_xml = ET.tostring(root, encoding="utf-8").decode("utf-8")
+        conn.defineXML(novo_xml)
+        print(f"[AMX VM] ✓ AMX ativado no XML da VM '{nome_vm}'. Reinicie a VM para aplicar.")
+        conn.close()
+        return True
+
+    except libvirt.libvirtError as e:
+        print(f"[AMX VM] Erro libvirt: {e}")
+        return False
+    except Exception as e:
+        print(f"[AMX VM] Erro inesperado: {e}")
+        return False
+
+
+# ============================================================================
+# Compatibility helpers (V5 API preserved)
+# ============================================================================
+
+def get_allocated_cores() -> int:
+    """Return the number of cores allocated by `optimize_xeon_environment`."""
+    return _NUCLEOS_ALOCADOS or 0
+
+
+def make_ort_session_options():
+    """Build onnxruntime.SessionOptions tuned for Xeon AVX512/AMX."""
+    import onnxruntime as ort  # type: ignore
+
+    n_cores = _NUCLEOS_ALOCADOS or _detect_physical_cores()
+
+    opts = ort.SessionOptions()
+    opts.intra_op_num_threads = n_cores
+    opts.inter_op_num_threads = 1
+    opts.graph_optimization_level = ort.GraphOptimizationLevel.ORT_ENABLE_ALL
+    opts.enable_cpu_mem_arena = True
+    return opts
+
+
+def get_xeon_status() -> Dict[str, Any]:
+    """V6: return a snapshot of all Xeon runtime flags + capabilities.
+
+    Useful for logging inside the training script.
+    """
+    amx_ok, amx_desc = get_amx_capability()
+    avx_ok, avx_desc = get_avx512_capability()
+    return {
+        "version": "V6",
+        "physical_cores": _NUCLEOS_ALOCADOS or _detect_physical_cores(),
+        "env": {
+            "MKL_ENABLE_INSTRUCTIONS": os.environ.get("MKL_ENABLE_INSTRUCTIONS"),
+            "MKL_NUM_THREADS": os.environ.get("MKL_NUM_THREADS"),
+            "OMP_NUM_THREADS": os.environ.get("OMP_NUM_THREADS"),
+            "MKL_DYNAMIC": os.environ.get("MKL_DYNAMIC"),
+            "DNNL_PRIMITIVE_CACHE_CAPACITY": os.environ.get("DNNL_PRIMITIVE_CACHE_CAPACITY"),
+            "ONEDNN_MAX_CPU_ISA": os.environ.get("ONEDNN_MAX_CPU_ISA"),
+            "KMP_AFFINITY": os.environ.get("KMP_AFFINITY"),
+            "KMP_BLOCKTIME": os.environ.get("KMP_BLOCKTIME"),
+        },
+        "avx512": {"supported": avx_ok, "desc": avx_desc},
+        "amx": {"supported": amx_ok, "desc": amx_desc},
+        "ipex_available": _IPEX_AVAILABLE is True,
+        "init_done": _V6_INIT_DONE,
+    }
+
+
+__all__ = [
+    "optimize_xeon_environment",
+    "apply_ipex_optimization",
+    "fp16_autocast",
+    "benchmark_fp16_matmul",
+    "benchmark_int8_matmul",
+    "ativar_amx_na_vm",
+    "get_avx512_capability",
+    "get_amx_capability",
+    "get_xeon_status",
+    "get_allocated_cores",
+    "make_ort_session_options",
+]