V6.5-V2-unified: cleanup misplaced root files + old scripts [18 files deleted] — removes duplicates from repo root (they live in src/bigru_t/...) and outdated scripts in scripts/ (now in deprecados/)

attention_multimodal.py

@@ -1,114 +0,0 @@
-"""
-Xavante - attention_multimodal.py
-Responsabilidade: Multi-head attention multimodal (Teorema 11.1/11.2).
-Suporta attention entre modalidades e dentro de modalidade.
-"""
-from __future__ import annotations
-
-import logging
-import math
-from typing import Optional
-
-import torch
-import torch.nn as nn
-import torch.nn.functional as F
-
-logger = logging.getLogger(__name__)
-
-
-class MultiHeadAttention(nn.Module):
-    """
-    MHA padrão com suporte a:
-    - Mascara causal
-    - Mascara multimodal (mod crossing)
-    - Attention 2M tokens via chunked attention (Teorema 6.1)
-    """
-
-    def __init__(
-        self,
-        d_model: int,
-        n_heads: int = 8,
-        dropout: float = 0.0,
-        max_chunk: int = 4096,
-    ):
-        super().__init__()
-        assert d_model % n_heads == 0
-        self.d_model = d_model
-        self.n_heads = n_heads
-        self.d_head = d_model // n_heads
-        self.max_chunk = max_chunk
-        self.qkv = nn.Linear(d_model, 3 * d_model, bias=True)
-        self.out = nn.Linear(d_model, d_model)
-        self.dropout = nn.Dropout(dropout)
-
-    def forward(
-        self,
-        x: torch.Tensor,
-        mask: Optional[torch.Tensor] = None,
-        kv: Optional[torch.Tensor] = None,
-    ) -> torch.Tensor:
-        B, L, D = x.shape
-        if kv is None:
-            qkv = self.qkv(x)
-            q, k, v = qkv.chunk(3, dim=-1)
-        else:
-            q = self.qkv(x)[:, :, :D]
-            kv_proj = self.qkv(kv)
-            k = kv_proj[:, :, D : 2 * D]
-            v = kv_proj[:, :, 2 * D :]
-        # reshape para heads
-        q = q.view(B, L, self.n_heads, self.d_head).transpose(1, 2)
-        k = k.view(B, -1, self.n_heads, self.d_head).transpose(1, 2)
-        v = v.view(B, -1, self.n_heads, self.d_head).transpose(1, 2)
-
-        # Chunked attention para sequencias longas (Teorema 6.1)
-        if L > self.max_chunk:
-            # Normalize mask to 4D for chunked path
-            if mask is not None and mask.dim() == 2:
-                mask = mask.unsqueeze(0).unsqueeze(0)
-            elif mask is not None and mask.dim() == 3:
-                mask = mask.unsqueeze(1)
-            return self._chunked_attention(q, k, v, mask)
-
-        scale = 1.0 / math.sqrt(self.d_head)
-        attn = (q @ k.transpose(-2, -1)) * scale  # [B, H, L, L]
-        if mask is not None:
-            attn = attn.masked_fill(mask == 0, float("-inf"))
-        attn = F.softmax(attn, dim=-1)
-        attn = self.dropout(attn)
-        out = attn @ v  # [B, H, L, d_head]
-        out = out.transpose(1, 2).contiguous().view(B, L, D)
-        return self.out(out)
-
-    def _chunked_attention(
-        self,
-        q: torch.Tensor,
-        k: torch.Tensor,
-        v: torch.Tensor,
-        mask: Optional[torch.Tensor],
-    ) -> torch.Tensor:
-        """Attention em blocos para janelas de 2M tokens (memoria controlada)."""
-        B, H, L, d = q.shape
-        Lk = k.shape[2]
-        chunk = self.max_chunk
-        outs = []
-        scale = 1.0 / math.sqrt(d)
-        for i in range(0, L, chunk):
-            qi = q[:, :, i : i + chunk]
-            out_chunk = []
-            for j in range(0, Lk, chunk):
-                kj = k[:, :, j : j + chunk]
-                vj = v[:, :, j : j + chunk]
-                attn = (qi @ kj.transpose(-2, -1)) * scale
-                if mask is not None:
-                    m_chunk = mask[:, :, i : i + chunk, j : j + chunk]
-                    attn = attn.masked_fill(m_chunk == 0, float("-inf"))
-                attn = F.softmax(attn, dim=-1)
-                out_chunk.append(attn @ vj)
-            outs.append(torch.cat(out_chunk, dim=2))
-        out = torch.cat(outs, dim=2)
-        out = out.transpose(1, 2).contiguous().view(B, L, self.n_heads * d)
-        return self.out(out)
-
-
-__all__ = ["MultiHeadAttention"]

hyp_t.py

@@ -1,100 +0,0 @@
-"""hyp_t.py — HypT: cabeça de hipótese (correção delta). Implementa o Lema 3.
-
-A Hipótese é treinada para cancelar o ruído de quantização W8A8:
-    y_final = y_hat + delta
-onde delta = hyp_T(o).
-
-Diferenças vs. TrainT:
-- Recebe `o` opcionalmente destacado (stop_grad) para isolar o sinal de
-  correção do gradiente principal
-- Mesma arquitetura (TransformerEncoder) — apenas o tratamento de gradiente
-  difere
-
-Reaproveita a filosofia do HypothesisNetwork do v13.9.2 (16 hipóteses com
-mapeamento LoRA), mas simplificada: 1 única hipótese global (delta) em vez
-de 16 hipóteses paralelas. A multiplexação é feita pelo OrqCell + Lema 1.
-
-============================================================================
-V6.4 — pgvector_lookup REMOVIDO (NÃO É MAIS NECESSÁRIO)
-============================================================================
-Conforme directive do usuário:
-  "pgvector_lookup não é mais necessário pela lógica do script seguinte"
-
-O KohonenLearningSystem (kohonen_refactored/kohonen_learning_system.py)
-contém o método find_bmu que realiza busca nearest-neighbor sobre o grid
-4D do SOM (864 neurônios para grid (6,6,6,4)), substituindo qualquer
-lookup pgvector externo. A decisão de aplicar punição é delegada ao
-protocolo interno do KohonenLearningSystem (1ª punição → activate_hypothesis,
-2ª punição → set_ewc_reference + reset).
-
-Consequentemente, HypT volta ao seu papel original: gerar a correção
-delta = hyp_T(o) sem consultar mapa externo. A integração com o SOM é
-feita no nível do trainer (KohonenLearningSystem.process_batch), não no
-nível da hipótese.
-
-Os contadores de punição (n_complete_skip, n_incomplete_apply, etc.)
-foram removidos pois não há mais decisão de skip baseada em pgvector.
-"""
-from __future__ import annotations
-
-import torch
-import torch.nn as nn
-
-
-class HypT(nn.Module):
-    """Transformer de hipótese: gera correção delta.
-
-    Args:
-        d_input: dimensão de entrada (saída do OrqCell = d_cache)
-        d_model: dimensão interna
-        nhead: nº de cabeças
-        d_ff: dimensão do FFN
-        output_dim: dimensão de saída (igual ao TrainT)
-        num_layers: nº de camadas
-        dropout: probabilidade de dropout
-
-    Forward:
-        o: (batch, d_input), stop_grad: bool → delta: (batch, output_dim)
-
-    Se stop_grad=True, aplica o.detach() no início — Lema 3 (isolar correção
-    do gradiente principal).
-    """
-
-    def __init__(
-        self,
-        d_input: int,
-        d_model: int,
-        nhead: int,
-        d_ff: int,
-        output_dim: int,
-        num_layers: int = 2,
-        dropout: float = 0.1,
-    ):
-        super().__init__()
-        self.d_input = d_input
-        self.d_model = d_model
-        self.output_dim = output_dim
-
-        self.proj_in = nn.Linear(d_input, d_model)
-        encoder_layer = nn.TransformerEncoderLayer(
-            d_model, nhead, d_ff, dropout=dropout, batch_first=True
-        )
-        self.encoder = nn.TransformerEncoder(encoder_layer, num_layers)
-        self.proj_out = nn.Linear(d_model, output_dim)
-
-    def forward(self, o: torch.Tensor, stop_grad: bool = True) -> torch.Tensor:
-        """o: (batch, d_input) → delta: (batch, output_dim)
-
-        V6.4: SEM consulta pgvector — delta é computado diretamente.
-        A decisão de aplicar ou não a correção é delegada ao
-        KohonenLearningSystem (que usa find_bmu internamente).
-        """
-        # Lema 3: stop_grad_hyp isola o sinal de correção
-        if stop_grad:
-            o = o.detach()
-
-        x = o.unsqueeze(1)               # (batch, 1, d_input)
-        x = self.proj_in(x)              # (batch, 1, d_model)
-        x = self.encoder(x)              # (batch, 1, d_model)
-        delta = self.proj_out(x.squeeze(1))  # (batch, output_dim)
-        return delta

scripts/train_v6_4.py

@@ -1,600 +0,0 @@
-"""train_v6_4.py — V6.4 REFACTORED Kohonen Learning System (pgvector_lookup REMOVED).
-
-═══════════════════════════════════════════════════════════════════════════════
-V6.4 — REFATORAÇÃO CANÔNICA COM CÓDIGO FORNECIDO PELO USUÁRIO
-═══════════════════════════════════════════════════════════════════════════════
-
-Refatoração sobrescreve os módulos usando o código Kohonen SOM 4D fornecido
-pelo usuário (versão limpa, find_bmu já corrigido). Análise matemática
-formal documentada em kohonen_learning_system.py.
-
-Diferenças vs V6.3:
-    1. pgvector_lookup REMOVIDO de hyp_t.py (não é mais necessário —
-       find_bmu do KohonenLearningSystem realiza a busca nearest-neighbor
-       sobre o grid 4D, substituindo qualquer lookup pgvector externo).
-    2. SOM grid放大ado para (6,6,6,4) = 864 neurônios (default do usuário).
-    3. hidden_dim=1024 (default do usuário, era 256 no V6.3).
-    4. T_max=10000 (default do usuário).
-    5. w = time_step / T_max (LINEAR no tempo, era sigmoid(||xyz||) no V6.3).
-    6. API unificada: kls.som.get_metrics() e kls.get_state_metrics().
-
-Componentes ativados:
-    1. xeon_runtime.py (AVX512 + AMX_INT8 + IPEX + OneDNN + FP16)
-    2. streaming_datasets.py (5 datasets × 50 samples = 250 total)
-    3. KohonenLearningSystem (refatorado, código CANÔNICO do usuário)
-    4. BATCH_SIZE = 16 (user requirement)
-    5. Captura de métricas: 12/12 + Kohonen (sigma, alpha, fisher) + Hyp
-    6. Monitoramento e informe de valores obtidos
-
-User requirements (V6.4):
-    - "refatorar sobrescrevendo os módulos usando (analisar matematicamente)"
-    - "pgvector_lookup não é mais necessário pela lógica do script seguinte"
-    - "HF_TOKEN=hf_<REDACTED_TOKEN> que deve ser apagada após uso"
-    - "usar streaming_datasets.py"
-    - "ativar xeon_runtime.py"
-    - "BATCH_SIZE = 16"
-    - "5 datasets × 50 samples streaming"
-    - "monitorar e informar valores obtidos"
-
-Saídas:
-    - /home/z/my-project/BiGRU_T_version/v6_4_report.json
-    - /home/z/my-project/BiGRU_T_version/v6_4_training_metrics.json
-    - Log no worklog.md
-═══════════════════════════════════════════════════════════════════════════════
-"""
-from __future__ import annotations
-
-import json
-import logging
-import os
-import sys
-import time
-import traceback
-from datetime import datetime
-from pathlib import Path
-from typing import Any, Dict, List, Optional
-
-# ============================================================================
-# 0. Paths e logging
-# ============================================================================
-PROJECT_ROOT = Path("/home/z/my-project")
-BIGRU_ROOT = PROJECT_ROOT / "BiGRU_T_version"
-SRC_ROOT = BIGRU_ROOT / "src"
-REPORT_PATH = BIGRU_ROOT / "v6_4_report.json"
-METRICS_PATH = BIGRU_ROOT / "v6_4_training_metrics.json"
-
-logging.basicConfig(
-    level=logging.INFO,
-    format="[%(asctime)s] [%(levelname)s] %(message)s",
-    datefmt="%H:%M:%S",
-)
-logger = logging.getLogger("train_v6_4")
-
-# ============================================================================
-# 1. ATIVAR xeon_runtime.py (user requirement, mencionado 2x)
-# ============================================================================
-sys.path.insert(0, str(SRC_ROOT))
-
-from bigru_t.utils.xeon_runtime import (  # noqa: E402
-    optimize_xeon_environment,
-    benchmark_fp16_matmul,
-    get_xeon_status,
-)
-
-N_CORES = optimize_xeon_environment(verbose=True)
-XEON_STATUS = get_xeon_status()
-FP16_BENCH = benchmark_fp16_matmul(size=4000, warmup=1, iters=2)
-logger.info(f"[V6.4] Xeon FP16 benchmark: {FP16_BENCH}")
-
-# ============================================================================
-# 2. Configurações V6.4 (defaults do código do usuário)
-# ============================================================================
-BATCH_SIZE = 16
-N_DATASETS = 5
-SAMPLES_PER_DATASET = 50
-TOTAL_SAMPLES = N_DATASETS * SAMPLES_PER_DATASET  # 250
-EPOCHS = 2
-MAX_SEQ_LEN = 8
-
-# V6.4: defaults canônicos do código do usuário
-HIDDEN_DIM = 1024          # era 256 no V6.3 — agora segue user spec
-VOCAB_SIZE = 16384         # default do SimpleBBPETokenizer
-SOM_GRID = (6, 6, 6, 4)    # 864 neurônios — default do usuário (era (2,2,2,1) no V6.3)
-T_MAX = 10000              # default do usuário
-N_START = 10               # default do usuário
-LAMBDA_EWC = 0.02          # default do usuário
-ALPHA0 = 0.1
-SIGMA0 = 1.5
-DIM_CHOICE = "y"
-
-V64_DATASETS = [
-    "TucanoBR/GigaVerbo",
-    "dominguesm/restore-punctuation-pttr-dataset",
-    "Madras1/corpus-ptbr-v2",
-    "CEIA-POSITIVO/ultrachat_br_clustred_balanced_v1",
-    "nvidia/OpenMathInstruct-2",
-]
-
-# Templates sintéticos para fallback (caso streaming falhe/lento)
-SYNTH_TEMPLATES = {
-    "TucanoBR/GigaVerbo": [
-        "o gato dorme na cama",
-        "o cachorro corre no parque",
-        "o pássaro voa no céu",
-        "a menina brinca com a boneca",
-        "o menino joga bola",
-    ],
-    "dominguesm/restore-punctuation-pttr-dataset": [
-        "o sol nasceu azul hoje",
-        "ela foi ao mercado comprar pão",
-        "nós viajamos para o rio de janeiro",
-        "o livro está sobre a mesa",
-        "a casa tem quatro quartos",
-    ],
-    "Madras1/corpus-ptbr-v2": [
-        "o brasil é um país tropical",
-        "a música popular brasileira é rica",
-        "o carnaval acontece em fevereiro",
-        "a floresta amazônica é vasta",
-        "o futebol é o esporte favorito",
-    ],
-    "CEIA-POSITIVO/ultrachat_br_clustred_balanced_v1": [
-        "olá como você está hoje",
-        "qual é o seu nome",
-        "pode me ajudar com isso",
-        "obrigado pela ajuda",
-        "até logo e boa noite",
-    ],
-    "nvidia/OpenMathInstruct-2": [
-        "dois mais dois igual a quatro",
-        "três vezes cinco é quinze",
-        "dez dividido por dois é cinco",
-        "sete menos três é quatro",
-        "oito mais nove é dezessete",
-    ],
-}
-
-# ============================================================================
-# 3. Import KohonenLearningSystem (V6.4 — refatorado canônico)
-# ============================================================================
-from bigru_t.model.kohonen_learning_system import (  # noqa: E402
-    KohonenLearningSystem,
-    SimpleBBPETokenizer,
-    positional_encoding,
-    text_to_4d_vector,
-    KohonenSOM4D,
-    HypothesisClassifier,
-)
-
-logger.info(
-    f"[V6.4] KohonenLearningSystem imported. "
-    f"Grid={SOM_GRID} ({SOM_GRID[0]*SOM_GRID[1]*SOM_GRID[2]*SOM_GRID[3]} neurons) | "
-    f"Hidden={HIDDEN_DIM} | Vocab={VOCAB_SIZE} | T_max={T_MAX}"
-)
-
-
-# ============================================================================
-# 4. Metrics Monitor (12/12 + Kohonen + Hypothesis — V6.4 sem pgvector)
-# ============================================================================
-class MetricsMonitorV64:
-    """Monitor 12/12 + Kohonen + Hypothesis para V6.4.
-
-    V6.4: SEM métricas de pgvector (removido conforme directive do usuário).
-    A decisão de aplicar punição é interna ao KohonenLearningSystem
-    (via find_bmu no SOM grid).
-    """
-
-    def __init__(self) -> None:
-        self.steps: List[Dict[str, Any]] = []
-        self.alerts: List[Dict[str, Any]] = []
-        self.start_time = time.time()
-        self._prev_loss: Optional[float] = None
-
-    def record_step(
-        self,
-        step: int,
-        epoch: int,
-        dataset_name: str,
-        batch_loss: float,
-        batch_acc: float,
-        kls: KohonenLearningSystem,
-        rss_mb: float,
-    ) -> None:
-        som_metrics = kls.som.get_metrics()
-        # Quality metrics (1.1-1.5)
-        quality = {
-            "1.1_train_loss": float(batch_loss),
-            "1.2_train_acc": float(batch_acc),
-            "1.3_val_loss": float(batch_loss),
-            "1.4_val_acc": float(batch_acc),
-            "1.5_perplexity": float(2.718281828 ** min(batch_loss, 20)),
-        }
-        # Speed metrics (2.1-2.5)
-        elapsed = time.time() - self.start_time
-        speed = {
-            "2.1_throughput_sps": float((step + 1) * BATCH_SIZE / max(elapsed, 1e-6)),
-            "2.2_step_time_ms": float(elapsed * 1000 / max(step + 1, 1)),
-            "2.3_epoch_progress": float(epoch + 1) / EPOCHS,
-            "2.4_rss_mb": float(rss_mb),
-            "2.5_xeon_tflops": float(FP16_BENCH.get("best_tflops", 0.0)),
-        }
-        # Kohonen metrics (V6.4 — API nova get_metrics)
-        kohonen = {
-            "sigma_t": float(som_metrics["sigma_t"]),
-            "alpha_t": float(som_metrics["alpha_t"]),
-            "t": int(som_metrics["t"]),
-            "n_neurons": int(som_metrics["n_neurons"]),
-            "fisher_w_mean": float(som_metrics["fisher_w_mean"]),
-            "fisher_w_max": float(som_metrics["fisher_w_max"]),
-            "fisher_accum_count": int(som_metrics["fisher_accum_count"]),
-            "has_ewc_reference": bool(som_metrics["has_ewc_reference"]),
-            "weights_norm": float(som_metrics["weights_norm"]),
-            "weights_w_mean": float(som_metrics["weights_w_mean"]),
-        }
-        # Hypothesis metrics (V6.4 — sem pgvector)
-        hyp = {
-            "classifier_trained": bool(kls.classifier_trained),
-            "punishment_count": int(kls.punishment_count),
-            "success_count": int(kls.success_count),
-            "training_ready": bool(kls.training_ready),
-            "buffer_size": int(len(kls.buffer_4d)),
-            "required_new_samples": int(kls.required_new_samples),
-            "histogram_max": int(max(kls.histogram.values(), default=0)),
-            "time_counter": int(kls.time_counter),
-        }
-        # Alerts (3.1-3.4)
-        if self._prev_loss is not None:
-            delta = abs(batch_loss - self._prev_loss)
-            if delta > 5.0:
-                self.alerts.append({
-                    "type": "3.1_loss_spike",
-                    "step": step,
-                    "delta": float(delta),
-                    "prev": float(self._prev_loss),
-                    "curr": float(batch_loss),
-                })
-            if batch_loss > 30.0:
-                self.alerts.append({
-                    "type": "3.2_loss_explosion",
-                    "step": step,
-                    "value": float(batch_loss),
-                })
-            if batch_loss < 0.001:
-                self.alerts.append({
-                    "type": "3.3_loss_vanishing",
-                    "step": step,
-                    "value": float(batch_loss),
-                })
-        self._prev_loss = float(batch_loss)
-        if rss_mb > 4096:
-            self.alerts.append({
-                "type": "3.4_rss_high",
-                "step": step,
-                "rss_mb": float(rss_mb),
-            })
-
-        self.steps.append({
-            "step": step,
-            "epoch": epoch,
-            "dataset": dataset_name,
-            "quality": quality,
-            "speed": speed,
-            "kohonen": kohonen,
-            "hypothesis": hyp,
-        })
-
-    def summary(self) -> Dict[str, Any]:
-        if not self.steps:
-            return {}
-        final = self.steps[-1]
-        losses = [s["quality"]["1.1_train_loss"] for s in self.steps]
-        accs = [s["quality"]["1.2_train_acc"] for s in self.steps]
-        sigmas = [s["kohonen"]["sigma_t"] for s in self.steps]
-        alphas = [s["kohonen"]["alpha_t"] for s in self.steps]
-        rss_max = max(s["speed"]["2.4_rss_mb"] for s in self.steps)
-        rss_final = final["speed"]["2.4_rss_mb"]
-        return {
-            "n_steps": len(self.steps),
-            "final_loss": float(losses[-1]),
-            "mean_loss": float(sum(losses) / len(losses)),
-            "min_loss": float(min(losses)),
-            "max_loss": float(max(losses)),
-            "final_acc": float(accs[-1]),
-            "mean_acc": float(sum(accs) / len(accs)),
-            "sigma_start": float(sigmas[0]),
-            "sigma_end": float(sigmas[-1]),
-            "alpha_start": float(alphas[0]),
-            "alpha_end": float(alphas[-1]),
-            "rss_max_mb": float(rss_max),
-            "rss_final_mb": float(rss_final),
-            "rss_trend": "stable" if abs(rss_max - rss_final) < 200 else "growing",
-            "n_alerts": len(self.alerts),
-            "alerts": self.alerts[:20],
-            "kohonen_final": final["kohonen"],
-            "hypothesis_final": final["hypothesis"],
-        }
-
-
-# ============================================================================
-# 5. Streaming dataset loader com fallback sintético
-# ============================================================================
-def load_streaming_samples(
-    dataset_name: str,
-    n_samples: int,
-    hf_token: Optional[str] = None,
-    timeout_s: int = 60,
-) -> List[str]:
-    """Carrega até n_samples de um dataset via streaming_datasets.
-
-    Fallback: se streaming falhar ou timeout, gera samples sintéticos.
-    """
-    samples: List[str] = []
-    t_start = time.time()
-    try:
-        from bigru_t.data.streaming_datasets import stream_dataset
-        for sample in stream_dataset(dataset_name, max_samples=n_samples, hf_token=hf_token):
-            if time.time() - t_start > timeout_s:
-                logger.warning(
-                    f"[V6.4] Streaming {dataset_name} timeout ({timeout_s}s) "
-                    f"after {len(samples)} samples"
-                )
-                break
-            if sample.raw_text and len(sample.raw_text.strip()) > 0:
-                samples.append(sample.raw_text.strip()[:200])
-            if len(samples) >= n_samples:
-                break
-    except Exception as e:
-        logger.warning(f"[V6.4] Streaming {dataset_name} failed: {e}")
-
-    if len(samples) < n_samples:
-        templates = SYNTH_TEMPLATES.get(dataset_name, ["exemplo genérico"])
-        needed = n_samples - len(samples)
-        logger.info(
-            f"[V6.4] Fallback sintético: gerando {needed} samples para {dataset_name} "
-            f"(streaming obteve {len(samples)})"
-        )
-        for i in range(needed):
-            base = templates[i % len(templates)]
-            samples.append(f"{base} (var {i})")
-    return samples[:n_samples]
-
-
-# ============================================================================
-# 6. Labels binárias (placeholder para classificação)
-# ============================================================================
-def make_label(text: str) -> int:
-    """Gera label binário determinístico baseado no texto."""
-    text_lower = text.lower()
-    if any(w in text_lower for w in ["gato", "mia", "dorme", "brinca", "menina", "boneca"]):
-        return 0
-    return 1
-
-
-# ============================================================================
-# 7. Função principal de treino
-# ============================================================================
-def main() -> int:
-    n_neurons = SOM_GRID[0] * SOM_GRID[1] * SOM_GRID[2] * SOM_GRID[3]
-    print("\n" + "=" * 76)
-    print("V6.4 — REFACTORED KOHONEN LEARNING SYSTEM (pgvector_lookup REMOVED)")
-    print("=" * 76)
-    print(f"  BATCH_SIZE           : {BATCH_SIZE}")
-    print(f"  Datasets             : {N_DATASETS}")
-    print(f"  Samples/dataset      : {SAMPLES_PER_DATASET}")
-    print(f"  Total samples        : {TOTAL_SAMPLES}")
-    print(f"  Epochs               : {EPOCHS}")
-    print(f"  SOM grid             : {SOM_GRID} ({n_neurons} neurons)")
-    print(f"  Hidden dim           : {HIDDEN_DIM}")
-    print(f"  Vocab size           : {VOCAB_SIZE}")
-    print(f"  T_max                : {T_MAX}")
-    print(f"  N_start              : {N_START}")
-    print(f"  lambda_ewc           : {LAMBDA_EWC}")
-    print(f"  Max seq len          : {MAX_SEQ_LEN}")
-    print(f"  Xeon cores           : {N_CORES}")
-    print(f"  Xeon AVX512          : {XEON_STATUS['avx512']['desc']}")
-    print(f"  Xeon AMX             : {XEON_STATUS['amx']['desc']}")
-    print(f"  Xeon IPEX            : {XEON_STATUS['ipex_available']}")
-    print(f"  FP16 best TFLOPS     : {FP16_BENCH.get('best_tflops', 0.0):.3f}")
-    print(f"  pgvector_lookup      : REMOVED (find_bmu replaces it)")
-    print("=" * 76 + "\n")
-
-    # Inicializa KohonenLearningSystem (V6.4 — defaults canônicos do usuário)
-    kls = KohonenLearningSystem(
-        vocab_size=VOCAB_SIZE,
-        hidden_dim=HIDDEN_DIM,
-        seq_len=MAX_SEQ_LEN,
-        som_grid=SOM_GRID,
-        alpha0=ALPHA0,
-        sigma0=SIGMA0,
-        lambda_ewc=LAMBDA_EWC,
-        N_start=N_START,
-        dim_choice=DIM_CHOICE,
-        hypothesis_hidden=[512, 256, 128, 64, 32, 16, 8],
-        T_max=T_MAX,
-    )
-    logger.info(
-        f"[V6.4] KohonenLearningSystem initialized. "
-        f"Vocab={VOCAB_SIZE}, Hidden={HIDDEN_DIM}, Grid={SOM_GRID} ({n_neurons} neurons)"
-    )
-
-    # Treina tokenizer com corpus sintético básico (PT-BR comum)
-    corpus_inicial = []
-    for templates in SYNTH_TEMPLATES.values():
-        corpus_inicial.extend(templates)
-    kls.tokenizer.fit(corpus_inicial)
-    logger.info(f"[V6.4] Tokenizer fitted with {len(corpus_inicial)} corpus words")
-
-    # Monitor
-    monitor = MetricsMonitorV64()
-
-    # HF_TOKEN (será apagado ao final)
-    hf_token = os.environ.get("HF_TOKEN")
-
-    # Loop de treino: 5 datasets × 50 samples × 2 epochs = 500 samples total
-    step = 0
-    t_train_start = time.time()
-
-    for epoch in range(EPOCHS):
-        logger.info(f"\n[V6.4] === Epoch {epoch + 1}/{EPOCHS} ===")
-        for ds_idx, dataset_name in enumerate(V64_DATASETS):
-            samples = load_streaming_samples(
-                dataset_name, SAMPLES_PER_DATASET, hf_token=hf_token, timeout_s=60
-            )
-            labels = [make_label(s) for s in samples]
-            # Processa em batches de BATCH_SIZE
-            for batch_start in range(0, len(samples), BATCH_SIZE):
-                batch_sents = samples[batch_start: batch_start + BATCH_SIZE]
-                batch_labels = labels[batch_start: batch_start + BATCH_SIZE]
-                try:
-                    stop_requested = kls.process_batch(batch_sents, batch_labels)
-                except Exception as e:
-                    logger.error(f"[V6.4] process_batch error: {e}")
-                    traceback.print_exc()
-                    continue
-
-                # Métricas
-                acc = kls.evaluate_classification()
-                # Loss proxy: -log(acc + eps) — menor acc = maior loss
-                loss = -max(0.01, acc) ** 0.5 if acc > 0 else 5.0
-                # RSS
-                try:
-                    import resource
-                    rss_kb = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss
-                    rss_mb = rss_kb / 1024.0
-                except (AttributeError, OSError):
-                    rss_mb = 0.0
-
-                monitor.record_step(
-                    step=step,
-                    epoch=epoch,
-                    dataset_name=dataset_name,
-                    batch_loss=float(loss),
-                    batch_acc=float(acc),
-                    kls=kls,
-                    rss_mb=float(rss_mb),
-                )
-                step += 1
-                if step % 5 == 0 or step == 1:
-                    som_m = kls.som.get_metrics()
-                    logger.info(
-                        f"[V6.4] step={step:3d} | ds={ds_idx+1}/{N_DATASETS} | "
-                        f"loss={loss:.3f} acc={acc:.3f} | "
-                        f"σ={som_m['sigma_t']:.3f} α={som_m['alpha_t']:.3f} | "
-                        f"punish={kls.punishment_count} success={kls.success_count} | "
-                        f"buff={len(kls.buffer_4d)} | "
-                        f"hyp={'Y' if kls.classifier_trained else 'N'} | "
-                        f"ewc={'Y' if som_m['has_ewc_reference'] else 'N'} | "
-                        f"fisher={som_m['fisher_w_mean']:.4f} | "
-                        f"RSS={rss_mb:.0f}MB"
-                    )
-                if stop_requested:
-                    logger.warning(
-                        f"[V6.4] stop_requested (2nd punishment → EWC reset) at step={step}. "
-                        f"Required new samples: {kls.required_new_samples}"
-                    )
-
-    t_train_end = time.time()
-    train_duration = t_train_end - t_train_start
-    logger.info(f"\n[V6.4] Treino concluído em {train_duration:.1f}s ({step} steps)")
-
-    # Finaliza e gera relatório
-    summary = monitor.summary()
-    som_final = kls.som.get_metrics()
-    kls_state = kls.get_state_metrics()
-
-    report = {
-        "version": "V6.4",
-        "timestamp": datetime.now().isoformat(),
-        "config": {
-            "BATCH_SIZE": BATCH_SIZE,
-            "N_DATASETS": N_DATASETS,
-            "SAMPLES_PER_DATASET": SAMPLES_PER_DATASET,
-            "TOTAL_SAMPLES": TOTAL_SAMPLES,
-            "EPOCHS": EPOCHS,
-            "SOM_GRID": list(SOM_GRID),
-            "n_neurons": n_neurons,
-            "HIDDEN_DIM": HIDDEN_DIM,
-            "VOCAB_SIZE": VOCAB_SIZE,
-            "MAX_SEQ_LEN": MAX_SEQ_LEN,
-            "T_max": T_MAX,
-            "N_start": N_START,
-            "lambda_ewc": LAMBDA_EWC,
-            "alpha0": ALPHA0,
-            "sigma0": SIGMA0,
-            "dim_choice": DIM_CHOICE,
-            "pgvector_lookup": "REMOVED (find_bmu replaces it)",
-        },
-        "xeon_status": XEON_STATUS,
-        "fp16_benchmark": FP16_BENCH,
-        "training": {
-            "duration_s": float(train_duration),
-            "n_steps": int(step),
-            "n_epochs": EPOCHS,
-        },
-        "summary": summary,
-        "kohonen_final": som_final,
-        "kls_state": kls_state,
-        "math_analysis": {
-            "text_to_4d": "SVD: M @ V[:3].T -> centroid 3D + w = time_step/T_max (LINEAR)",
-            "bmu_distance": "||W - x||^2 (L2 squared in R^4) — replaces pgvector_lookup",
-            "neighborhood": "Lambda(d, sigma) = exp(-d^2 / (2*sigma^2)), d^2 = di^2+dj^2+dk^2+dl^2",
-            "weight_update": "dW = alpha * Lambda * (x - W)",
-            "sigma_decay": "sigma_t = sigma0 * exp(-t/1000)",
-            "alpha_decay": "alpha_t = alpha0 * exp(-t/2000)",
-            "ewc_only_dim4": "penalty = lambda * F * (W_w - W*_w), F = mean((x_w - W_w)^2)",
-            "fisher_accumulation": "only when punishment_count==0 AND old_weights_w is None AND Lambda > 0.1",
-            "hypothesis_classifier": "8 layers FC: 512->256->128->64->32->16->8->1",
-            "punishment_protocol": "1st -> activate_hypothesis; 2nd -> set_ewc_reference + reset",
-            "bug_fixed_find_bmu": "user code already clean (no premature return)",
-            "bug_fixed_activate_hypothesis": "detach+clone buffer + no_grad for SOM activation (V6.3 fix maintained)",
-            "pgvector_removed": "find_bmu is the equivalent nearest-neighbor search over SOM grid",
-        },
-        "datasets_used": V64_DATASETS,
-    }
-
-    REPORT_PATH.write_text(json.dumps(report, indent=2, ensure_ascii=False))
-    logger.info(f"[V6.4] Report saved: {REPORT_PATH}")
-
-    metrics_full = {
-        "version": "V6.4",
-        "steps": monitor.steps,
-        "summary": summary,
-        "alerts": monitor.alerts,
-    }
-    METRICS_PATH.write_text(json.dumps(metrics_full, indent=2, ensure_ascii=False))
-    logger.info(f"[V6.4] Metrics saved: {METRICS_PATH}")
-
-    # Print final summary
-    print("\n" + "=" * 76)
-    print("V6.4 — TREINO CONCLUÍDO")
-    print("=" * 76)
-    print(f"  Steps              : {step}")
-    print(f"  Duration           : {train_duration:.1f}s")
-    print(f"  Final loss         : {summary.get('final_loss', 0):.3f}")
-    print(f"  Mean loss          : {summary.get('mean_loss', 0):.3f}")
-    print(f"  Final acc          : {summary.get('final_acc', 0):.3f}")
-    print(f"  Mean acc           : {summary.get('mean_acc', 0):.3f}")
-    print(f"  Sigma (start→end)  : {summary.get('sigma_start', 0):.3f} → {summary.get('sigma_end', 0):.3f}")
-    print(f"  Alpha (start→end)  : {summary.get('alpha_start', 0):.3f} → {summary.get('alpha_end', 0):.3f}")
-    print(f"  Fisher w mean      : {som_final['fisher_w_mean']:.6f}")
-    print(f"  Fisher w max       : {som_final['fisher_w_max']:.6f}")
-    print(f"  Fisher accum count : {som_final['fisher_accum_count']}")
-    print(f"  Weights norm       : {som_final['weights_norm']:.3f}")
-    print(f"  Weights w mean     : {som_final['weights_w_mean']:.6f}")
-    print(f"  RSS max            : {summary.get('rss_max_mb', 0):.0f}MB")
-    print(f"  RSS trend          : {summary.get('rss_trend', '?')}")
-    print(f"  Alerts             : {summary.get('n_alerts', 0)}")
-    print(f"  Hypothesis trained : {kls.classifier_trained}")
-    print(f"  EWC reference set  : {som_final['has_ewc_reference']}")
-    print(f"  Punishment count   : {kls.punishment_count}")
-    print(f"  Success count      : {kls.success_count}")
-    print(f"  Time counter       : {kls.time_counter}")
-    print("=" * 76)
-    print(f"\n  Report : {REPORT_PATH}")
-    print(f"  Metrics: {METRICS_PATH}\n")
-
-    return 0
-
-
-if __name__ == "__main__":
-    sys.exit(main())

scripts/upload_v6_5_7ds_attn_v3.py

@@ -1,373 +0,0 @@
-"""upload_v6_5_7ds_attn_v2.py — V6.5-attn-v2 upload to HuggingFace.
-
-User requirement: "upload dos arquivos testados e os aprimorados corrigidos
-(remover os antigos)"
-
-V6.5-attn-v2 = V6.5-attn + META ≥6000 + salvar estados + 3 perguntas reais sem ajuda.
-"""
-from __future__ import annotations
-import json, os, sys, time, re
-from pathlib import Path
-
-PROJECT_ROOT = Path("/home/z/my-project")
-BIGRU_ROOT = PROJECT_ROOT / "BiGRU_T_version"
-
-# Files to REMOVE (old V6.5-7ds files superseded by V6.5-attn + legacy files)
-OLD_FILES_TO_REMOVE = [
-    # ── Legacy root reports (v3/v4/v5/v6/v32) ─────────────────────────────────
-    "bug_hunt_v3_report.json",
-    "v3_report.json",
-    "v4_report.json",
-    "v5_report.json",
-    "v5_upload_report.json",
-    "v32_test_report.json",
-    "v32_upload_report.json",
-    "training_report.json",
-    "v6_report.json",
-    "v6_upload_report.json",
-    "v6_progressive_report.json",
-    "v6_progressive_train.log",
-    # ── Legacy v6.1/v6.2/v6.3 reports ─────────────────────────────────────────
-    "v6_1_report.json",
-    "v6_1_upload_report.json",
-    "v6_2_report.json",
-    "v6_2_upload_report.json",
-    "v6_3_report.json",
-    "v6_3_training_metrics.json",
-    # ── Old V6.5 reports (superseded) ─────────────────────────────────────────
-    "v6_5_report.json",
-    "v6_5_training_metrics.json",
-    "v6_5_module_analysis.json",
-    "v6_5_script_activity.json",
-    "v6_5_ewc_w8a8_benchmark.json",
-    "v6_5_reasoning_eval.json",
-    "v6_5_upload_report.json",
-    # ── Old V6.5-final reports (superseded) ───────────────────────────────────
-    "v6_5_final_report.json",
-    "v6_5_final_training_metrics.json",
-    "v6_5_final_module_analysis.json",
-    "v6_5_final_script_activity.json",
-    "v6_5_final_ewc_w8a8_benchmark.json",
-    "v6_5_final_reasoning_eval.json",
-    "v6_5_final_w8a8_compression.json",
-    # ── Old V6.5-7ds reports (superseded by V6.5-attn) ────────────────────────
-    "v6_5_7ds_report.json",
-    "v6_5_7ds_training_metrics.json",
-    "v6_5_7ds_module_analysis.json",
-    "v6_5_7ds_script_activity.json",
-    "v6_5_7ds_ewc_w8a8_benchmark.json",
-    "v6_5_7ds_reasoning_eval.json",
-    "v6_5_7ds_w8a8_compression.json",
-    "v6_5_7ds_upload_report.json",
-    # ── Legacy training scripts ───────────────────────────────────────────────
-    "scripts/parse_v6_log.py",
-    "scripts/smoke_test.py",
-    "scripts/train.py",
-    "scripts/train_fast.py",
-    "scripts/train_v2.py",
-    "scripts/train_v6_1.py",
-    "scripts/train_v6_2.py",
-    "scripts/train_v6_3.py",
-    "scripts/train_v6_progressive.py",
-    "scripts/train_v6_5.py",
-    "scripts/train_v6_5_final.py",
-    "scripts/train_v6_5_7ds.py",  # superseded by train_v6_5_7ds_attn.py
-    # ── Legacy upload scripts (only upload_v6_5_7ds_attn.py stays) ────────────
-    "scripts/upload_to_hf.py",
-    "scripts/upload_v6_1_resilient.py",
-    "scripts/upload_v6_2_resilient.py",
-    "scripts/upload_v6_3_resilient.py",
-    "scripts/upload_v6_4_resilient.py",
-    "scripts/upload_v6_5_resilient.py",
-    "scripts/upload_v6_5_7ds.py",  # superseded by upload_v6_5_7ds_attn.py
-    # ── Legacy tokenizer/ root config ─────────────────────────────────────────
-    "config.json",
-    "tokenizer/tokenizer.json",
-    # ── Legacy data_augmentation.py ───────────────────────────────────────────
-    "src/bigru_t/data/data_augmentation.py",
-    # ── Legacy xeon_runtime.py at scripts/ ────────────────────────────────────
-    "scripts/xeon_runtime.py",
-    # ── Old V6.5-7ds-attn v1 reports (superseded by V6.5-attn-v2) ────────────
-    "v6_5_7ds_attn_report.json",
-    "v6_5_7ds_attn_training_metrics.json",
-    "v6_5_7ds_attn_module_analysis.json",
-    "v6_5_7ds_attn_script_activity.json",
-    "v6_5_7ds_attn_ewc_w8a8_benchmark.json",
-    "v6_5_7ds_attn_reasoning_eval.json",
-    "v6_5_7ds_attn_w8a8_compression.json",
-    "v6_5_7ds_attn_attention_eval.json",
-    "v6_5_7ds_attn_verification.json",
-    "v6_5_7ds_attn_upload_report.json",
-    # ── Old V6.5-7ds-attn v1 scripts (superseded by v2) ──────────────────────
-    "scripts/train_v6_5_7ds_attn.py",
-    "scripts/upload_v6_5_7ds_attn.py",
-    # ── V6.5-fix: Old V6.5-7ds-attn v2 reports (superseded by v3) ────────────
-    # User requirement: "apagar v6_5_7ds_attn_v2_model_states.pt enviado ao HF
-    # devido a esse erro informado" (predict() retornava 'gato'/'cachorro'
-    # fixos ao invés de rótulos dinâmicos dos datasets).
-    "v6_5_7ds_attn_v2_report.json",
-    "v6_5_7ds_attn_v2_training_metrics.json",
-    "v6_5_7ds_attn_v2_module_analysis.json",
-    "v6_5_7ds_attn_v2_script_activity.json",
-    "v6_5_7ds_attn_v2_ewc_w8a8_benchmark.json",
-    "v6_5_7ds_attn_v2_reasoning_eval.json",
-    "v6_5_7ds_attn_v2_w8a8_compression.json",
-    "v6_5_7ds_attn_v2_attention_eval.json",
-    "v6_5_7ds_attn_v2_user_questions.json",
-    "v6_5_7ds_attn_v2_model_states.pt",  # ← BUG: predict() retornava gato/cachorro
-    "v6_5_7ds_attn_v2_upload_report.json",
-    # ── V6.5-fix: Old V6.5-7ds-attn v2 scripts (superseded by v3) ────────────
-    "scripts/train_v6_5_7ds_attn_v2.py",
-    "scripts/upload_v6_5_7ds_attn_v2.py",
-]
-
-# Critical files that MUST be present after upload
-CRITICAL_FILES_V65_ATTN = [
-    # Core model
-    "src/bigru_t/model/kohonen_learning_system.py",
-    "src/bigru_t/model/__init__.py",
-    "src/bigru_t/__init__.py",
-    "src/bigru_t/model/hyp_t.py",
-    "src/bigru_t/model/vqvae2_hierarchical.py",
-    "src/bigru_t/model/vqvae2_hierarchical_flexnet.py",
-    "src/bigru_t/model/token_compress.py",
-    "src/bigru_t/model/embedding_reconfig.py",
-    "src/bigru_t/model/attention_multimodal.py",
-    "src/bigru_t/attention/window_context.py",
-    # Training
-    "src/bigru_t/training/mtp.py",
-    "src/bigru_t/training/ewc.py",
-    # Quantization
-    "src/bigru_t/quantization/smoothquant_compressor.py",
-    "src/bigru_t/quantization/w8a8_smoothquant.py",
-    "src/bigru_t/quantization/quantized_linear.py",
-    # Reasoning
-    "src/bigru_t/reasoning/thinking.py",
-    "src/bigru_t/reasoning/reasoning_engine.py",
-    "src/bigru_t/reasoning/circular_orchestration.py",
-    "src/bigru_t/reasoning/tool_agent.py",
-    "src/bigru_t/reasoning/distributed_reasoning_system.py",
-    "src/bigru_t/reasoning/cyclic_reasoning.py",
-    "src/bigru_t/reasoning/consensus_sampling.py",
-    # Data + utils
-    "src/bigru_t/data/streaming_datasets.py",
-    "src/bigru_t/utils/xeon_runtime.py",
-    # V6.5-attn-v3 reports (NEW canonical — V6.5-fix: dynamic labels + inference punishment)
-    "v6_5_7ds_attn_v3_report.json",
-    "v6_5_7ds_attn_v3_training_metrics.json",
-    "v6_5_7ds_attn_v3_module_analysis.json",
-    "v6_5_7ds_attn_v3_script_activity.json",
-    "v6_5_7ds_attn_v3_ewc_w8a8_benchmark.json",
-    "v6_5_7ds_attn_v3_reasoning_eval.json",
-    "v6_5_7ds_attn_v3_w8a8_compression.json",
-    "v6_5_7ds_attn_v3_attention_eval.json",
-    "v6_5_7ds_attn_v3_user_questions.json",
-    "v6_5_7ds_attn_v3_predict_fix_eval.json",
-    "v6_5_7ds_attn_v3_inference_punishment.json",
-    "v6_5_7ds_attn_v3_model_states.pt",
-    "v6_5_7ds_attn_v3_upload_report.json",
-    # V6.4 reports (kept as predecessor baseline)
-    "v6_4_report.json",
-    "v6_4_training_metrics.json",
-    "v6_4_upload_report.json",
-    # V6.5-attn-v3 scripts (the new canonicals)
-    "scripts/train_v6_5_7ds_attn_v3.py",
-    "scripts/upload_v6_5_7ds_attn_v3.py",
-    "scripts/train_v6_4.py",
-    # Misc
-    "requirements.txt",
-    "README.md",
-    "docs/analysis.md",
-]
-
-
-def main() -> int:
-    print("=" * 72)
-    print("V6.5-attn-v3 — UPLOAD TO HUGGINGFACE (dynamic labels + inference punishment)")
-    print("=" * 72)
-    hf_token = os.environ.get("HF_TOKEN")
-    if not hf_token:
-        print("ERROR: HF_TOKEN not set in env")
-        return 1
-    print(f"  HF_TOKEN loaded from env (length={len(hf_token)})")
-
-    # ── Step 1: Pre-upload safety — scrub any HF token from scripts ──────────
-    token_pattern = re.compile(r'hf_[A-Za-z0-9]{32,}')
-    print("\n  [1/5] Pre-upload safety: scrubbing token from scripts...")
-    scrubbed_count = 0
-    for script_path in BIGRU_ROOT.glob("scripts/*.py"):
-        try:
-            content = script_path.read_text()
-            if token_pattern.search(content):
-                scrubbed = token_pattern.sub("hf_<REDACTED_TOKEN>", content)
-                script_path.write_text(scrubbed)
-                scrubbed_count += 1
-                print(f"    SCRUBBED: {script_path.name}")
-        except Exception as e:
-            print(f"    SKIP {script_path.name}: {e}")
-    if scrubbed_count == 0:
-        print("    (no token leakage found in scripts)")
-
-    # Also scrub from KLS source files
-    print("    Also scrubbing src/bigru_t/...")
-    for src_path in (BIGRU_ROOT / "src" / "bigru_t").rglob("*.py"):
-        try:
-            content = src_path.read_text()
-            if token_pattern.search(content):
-                scrubbed = token_pattern.sub("hf_<REDACTED_TOKEN>", content)
-                src_path.write_text(scrubbed)
-                scrubbed_count += 1
-                print(f"    SCRUBBED: {src_path.relative_to(BIGRU_ROOT)}")
-        except Exception:
-            pass
-
-    # ── Step 2: Import HF API + check repo access ────────────────────────────
-    print("\n  [2/5] Connecting to HuggingFace repo...")
-    try:
-        from huggingface_hub import HfApi, upload_folder
-    except ImportError:
-        print("ERROR: huggingface_hub not installed")
-        return 1
-
-    api = HfApi(token=hf_token)
-    repo_id = "PowerMachine/BiGRU_T_version"
-    try:
-        info = api.repo_info(repo_id=repo_id, repo_type="model")
-        print(f"  Repo: {repo_id} (existing, files={len(info.siblings)})")
-    except Exception as e:
-        print(f"  ERROR: cannot access repo: {e}")
-        return 1
-
-    files_before = set(s.rfilename for s in info.siblings)
-    print(f"  Files in repo BEFORE upload: {len(files_before)}")
-
-    # ── Step 3: Upload current BiGRU_T_version tree as single commit ─────────
-    print(f"\n  [3/5] Uploading {BIGRU_ROOT} as single commit...")
-    t0 = time.time()
-    try:
-        commit_info = upload_folder(
-            repo_id=repo_id, repo_type="model",
-            folder_path=str(BIGRU_ROOT),
-            commit_message=(
-                "V6.5-attn-v3: CORREÇÃO MATEMÁTICA do predict() — rótulos dinâmicos "
-                "extraídos do label_registry (substitui 'gato'/'cachorro' fixos). "
-                "NOVO: punish_during_inference() — mirror do protocolo de punição "
-                "do treinamento (1ª → activate_hypothesis, 2ª → set_ewc_reference). "
-                "META ≥6000 samples, attention ACTIVE_AND_FUNCTIONAL, SmoothQuant W8A8, "
-                "864 neurons, MTP K=6, 3 perguntas reais sem ajuda (Luva de Pedreiro "
-                "Távila, Lula reserva valor, Amazonas força-tarefa vítimas) + punição "
-                "na inferência via camada de hipótese. "
-                "Removed legacy v2 reports/scripts (superseded by v3)."
-            ),
-            token=hf_token,
-        )
-        t1 = time.time()
-        print(f"  Upload completed in {t1 - t0:.2f}s")
-        print(f"  Commit: {commit_info.oid}")
-        print(f"  URL: https://huggingface.co/PowerMachine/BiGRU_T_version/commit/{commit_info.oid}")
-    except Exception as e:
-        print(f"  ERROR: upload failed: {e}")
-        return 1
-
-    # ── Step 4: Remove OLD files from repo (single delete commit) ────────────
-    print(f"\n  [4/5] Removing {len(OLD_FILES_TO_REMOVE)} old files from repo...")
-    try:
-        info_after = api.repo_info(repo_id=repo_id, repo_type="model")
-        files_after_upload = set(s.rfilename for s in info_after.siblings)
-    except Exception as e:
-        print(f"  WARNING: cannot refresh repo info: {e}")
-        files_after_upload = files_before
-
-    old_files_in_repo = [f for f in OLD_FILES_TO_REMOVE if f in files_after_upload]
-    old_files_not_in_repo = [f for f in OLD_FILES_TO_REMOVE if f not in files_after_upload]
-    print(f"    Old files present in repo (will delete): {len(old_files_in_repo)}")
-    print(f"    Old files NOT in repo (skip): {len(old_files_not_in_repo)}")
-
-    delete_errors = []
-    commit_del = None
-    if old_files_in_repo:
-        try:
-            from huggingface_hub import CommitOperation
-            operations = [CommitOperationDelete(path_in_repo=f) for f in old_files_in_repo]
-            print(f"    Creating batch delete commit ({len(operations)} files)...")
-            t_del_start = time.time()
-            commit_del = api.create_commit(
-                repo_id=repo_id,
-                repo_type="model",
-                operations=operations,
-                commit_message=(
-                    f"V6.5-attn-v2 cleanup: remove {len(operations)} legacy/superseded files "
-                    f"(v3/v4/v5/v6_1/v6_2/v6_3/v6_progressive + old v6_5_*/v6_5_final_*/v6_5_7ds_*/v6_5_7ds_attn_* v1 "
-                    f"reports/scripts) — replaced by V6.5-attn-v2 canonicals"
-                ),
-            )
-            t_del_end = time.time()
-            print(f"    Delete commit: {commit_del}")
-            print(f"    Delete completed in {t_del_end - t_del_start:.2f}s")
-            print(f"    URL: https://huggingface.co/PowerMachine/BiGRU_T_version/commit/{commit_del}")
-        except Exception as e:
-            print(f"  ERROR: batch delete failed: {e}")
-            print(f"  Falling back to per-file delete...")
-            for f in old_files_in_repo:
-                try:
-                    api.delete_file(
-                        repo_id=repo_id, repo_type="model",
-                        path_in_repo=f,
-                        commit_message=f"V6.5-attn-v2 cleanup: remove legacy {f}",
-                        token=hf_token,
-                    )
-                    print(f"    DELETED: {f}")
-                except Exception as e2:
-                    delete_errors.append((f, str(e2)[:100]))
-                    print(f"    FAILED: {f} — {str(e2)[:100]}")
-
-    # ── Step 5: Verify critical files in repo ────────────────────────────────
-    print(f"\n  [5/5] Verifying critical V6.5-attn files in repo...")
-    all_present = True
-    files_in_repo = set()
-    try:
-        info_final = api.repo_info(repo_id=repo_id, repo_type="model")
-        files_in_repo = set(s.rfilename for s in info_final.siblings)
-        for cf in CRITICAL_FILES_V65_ATTN:
-            present = cf in files_in_repo
-            mark = "OK" if present else "MISS"
-            print(f"    [{mark}] {cf}")
-            if not present:
-                all_present = False
-        print(f"\n  Files in repo AFTER cleanup: {len(files_in_repo)}")
-        if not all_present:
-            print("  WARNING: some critical files missing!")
-    except Exception as e:
-        print(f"  WARNING: cannot verify files: {e}")
-
-    # ── Save upload report ───────────────────────────────────────────────────
-    report = {
-        "version": "V6.5-attn-v3",
-        "upload_timestamp": time.strftime("%Y-%m-%dT%H:%M:%S"),
-        "repo_id": repo_id,
-        "upload_commit_oid": commit_info.oid,
-        "upload_commit_url": f"https://huggingface.co/PowerMachine/BiGRU_T_version/commit/{commit_info.oid}",
-        "upload_duration_s": float(t1 - t0),
-        "delete_commit_oid": commit_del,
-        "delete_count": len(old_files_in_repo) if old_files_in_repo else 0,
-        "delete_errors": delete_errors,
-        "n_files_before": len(files_before),
-        "n_files_after_upload": len(files_after_upload),
-        "n_files_after_cleanup": len(files_in_repo) if files_in_repo else None,
-        "old_files_removed": old_files_in_repo,
-        "old_files_not_in_repo_skipped": old_files_not_in_repo,
-        "critical_files": CRITICAL_FILES_V65_ATTN,
-        "all_critical_files_present": all_present,
-        "token_scrubbed_count": scrubbed_count,
-    }
-    report_path = BIGRU_ROOT / "v6_5_7ds_attn_v3_upload_report.json"
-    report_path.write_text(json.dumps(report, indent=2, ensure_ascii=False))
-    print(f"\n  Upload report: {report_path}")
-    print("\n" + "=" * 72)
-    print("V6.5-attn-v3 — UPLOAD COMPLETED (with old file removal)")
-    print("=" * 72)
-    return 0
-
-
-if __name__ == "__main__":
-    sys.exit(main())

som_metrics.py

@@ -1,993 +0,0 @@
-"""som_metrics.py — V6.5-V2-metrics: Métricas de aprendizado e indicadores de
-falha para Mapas Auto-Organizáveis (SOM / Redes de Kohonen).
-
-============================================================================
-MÉTRICAS PRINCIPAIS DE APRENDIZADO
-============================================================================
-
-1.1. Erro de Quantização (Quantization Error - QE)
-     Média das distâncias euclidianas entre cada vetor de entrada e o vetor
-     de pesos de seu neurônio vencedor (BMU). Mede a resolução ou ajuste fino
-     do mapa aos dados.
-
-         QE = (1/N) * Σ_i ||x_i - W_BMU(x_i)||₂
-
-1.2. Erro Topológico (Topological Error - TE)
-     Percentual de vetores de entrada cujos primeiro e segundo neurônios
-     mais próximos (BMU e segunda BMU) NÃO são vizinhos diretos na grade
-     de saída. Avalia preservação das relações geométricas.
-
-         TE = (1/N) * Σ_i 𝟙[¬adjacent(BMU_1(x_i), BMU_2(x_i))]
-
-1.3. Erro de Kaski-Lagus
-     Combinação ponderada que penaliza tanto a falta de resolução do vetor
-     de quantização quanto as quebras na continuidade topológica.
-
-         KL = α * QE_norm + β * TE
-         onde α=0.5, β=0.5 (pesos padrão)
-
-     Variante implementada: KL = QE * (1 + TE)  (penaliza QE quando TE é alto)
-
-1.4. Variância Explicada (Share of Explained Variance)
-     Fração da variabilidade total dos dados de entrada que é mapeada e
-     representada pelos vetores de referência (pesos) dos nós da rede.
-
-         VarExplained = 1 - Var(residual) / Var(total)
-         onde residual = x_i - W_BMU(x_i)
-
-============================================================================
-INDICADORES DE NÃO APRENDIZADO OU FALHA
-============================================================================
-
-2.1. Colapso Topológico (Topological Collapse)
-     Ocorre quando todos os vetores de peso convergem para o mesmo ponto
-     ou para uma linha reta. Indica perda total da dimensionalidade útil.
-
-     Detecção:
-       - std(weights) por dimensão < threshold (colapso total)
-       - rank(weights reshaped) < 2 (todos num ponto)
-       - rank < 3 (todos numa linha)
-
-2.2. Taxa de Neurônios Mortos (Dead Neurons)
-     Percentual de neurônios que NUNCA se tornam BMU para nenhum vetor.
-
-         DeadRate = (n_neurons - n_active_neurons) / n_neurons
-
-     Threshold: DeadRate > 0.5 indica problema.
-
-2.3. Estagnação do Erro de Quantização
-     QE estabiliza em patamar elevado ao longo das épocas. A taxa de
-     aprendizado decaiu rápido demais antes da fase de ordenação global.
-
-     Detecção: slope(QE) ≈ 0 E QE atual > 0.7 * max(QE histórico)
-
-2.4. Cruzamento de Vizinhança (Neighborhood Crossing)
-     Crescimento repentino ou oscilação crônica do TE nas épocas finais.
-
-     Detecção:
-       - std(TE_final_5_epochs) > 2 * std(TE_initial_5_epochs) (oscilação)
-       - TE[-1] > TE[-5] (crescimento no final)
-
-============================================================================
-AUTOR: V6.5-V2-metrics
-============================================================================
-"""
-from __future__ import annotations
-
-import math
-import logging
-from collections import defaultdict, deque
-from dataclasses import dataclass, field
-from typing import Any, Dict, List, Optional, Tuple
-
-import torch
-import torch.nn.functional as F
-
-logger = logging.getLogger(__name__)
-
-
-# ============================================================================
-# Dataclass para armazenar histórico de métricas
-# ============================================================================
-@dataclass
-class SOMMetricHistory:
-    """Histórico deslizante de métricas SOM para detecção de estagnação e
-    cruzamento de vizinhança."""
-    qe_history: deque = field(default_factory=lambda: deque(maxlen=50))
-    te_history: deque = field(default_factory=lambda: deque(maxlen=50))
-    kl_history: deque = field(default_factory=lambda: deque(maxlen=50))
-    var_explained_history: deque = field(default_factory=lambda: deque(maxlen=50))
-    dead_neuron_rate_history: deque = field(default_factory=lambda: deque(maxlen=50))
-    epoch: int = 0
-
-    @staticmethod
-    def _safe_float(v: Any, default: float = 0.0) -> float:
-        """Converte para float, substituindo NaN/Inf por default.
-
-        V6.5-V2-metrics-FIX-3: Um único NaN no histórico contamina
-        permanentemente a detecção de estagnação (regressão linear)
-        e de cruzamento de vizinhança (média/desvio-padrão). Esta
-        barreira garante que somente valores finitos entrem no deque.
-        """
-        try:
-            f = float(v)
-        except (TypeError, ValueError):
-            return default
-        if not math.isfinite(f):
-            return default
-        return f
-
-    def record(self, metrics: Dict[str, Any]) -> None:
-        """Registra métricas de uma época (NaN/Inf são substituídos por 0.0).
-
-        Nota: `dead_neuron_rate` pode ser float (valor direto) ou Dict
-        (saída de compute_all_metrics, com chave 'dead_neuron_rate').
-        """
-        self.qe_history.append(self._safe_float(metrics.get("quantization_error", 0.0)))
-        self.te_history.append(self._safe_float(metrics.get("topological_error", 0.0)))
-        self.kl_history.append(self._safe_float(metrics.get("kaski_lagus_error", 0.0)))
-        self.var_explained_history.append(
-            self._safe_float(metrics.get("explained_variance_share", 0.0))
-        )
-        # dead_neuron_rate pode ser float ou Dict
-        dnr = metrics.get("dead_neuron_rate", 0.0)
-        if isinstance(dnr, dict):
-            dnr_val = self._safe_float(dnr.get("dead_neuron_rate", 0.0))
-        else:
-            dnr_val = self._safe_float(dnr)
-        self.dead_neuron_rate_history.append(dnr_val)
-        self.epoch += 1
-
-    def to_dict(self) -> Dict[str, Any]:
-        return {
-            "epoch": int(self.epoch),
-            "qe_history": list(self.qe_history),
-            "te_history": list(self.te_history),
-            "kl_history": list(self.kl_history),
-            "var_explained_history": list(self.var_explained_history),
-            "dead_neuron_rate_history": list(self.dead_neuron_rate_history),
-        }
-
-
-# ============================================================================
-# Funções auxiliares
-# ============================================================================
-def _grid_neighbors_set(
-    idx: Tuple[int, int, int, int],
-    grid_shape: Tuple[int, int, int, int],
-) -> set:
-    """Retorna o conjunto de índices vizinhos diretos (6-connectivity em 4D)
-    de um neurônio na grade."""
-    i, j, k, l = idx
-    I, J, K, L = grid_shape
-    neighbors = set()
-    for di, dj, dk, dl in [
-        (1, 0, 0, 0), (-1, 0, 0, 0),
-        (0, 1, 0, 0), (0, -1, 0, 0),
-        (0, 0, 1, 0), (0, 0, -1, 0),
-        (0, 0, 0, 1), (0, 0, 0, -1),
-    ]:
-        ni, nj, nk, nl = i + di, j + dj, k + dk, l + dl
-        if 0 <= ni < I and 0 <= nj < J and 0 <= nk < K and 0 <= nl < L:
-            neighbors.add((ni, nj, nk, nl))
-    return neighbors
-
-
-def _find_bmu_and_second(
-    x: torch.Tensor, weights: torch.Tensor
-) -> Tuple[Tuple[int, int, int, int], Tuple[int, int, int, int], float, float]:
-    """Encontra BMU e 2ª BMU para um vetor x.
-
-    Args:
-        x: tensor [4] — vetor 4D de entrada.
-        weights: tensor [I, J, K, L, 4] — pesos do SOM.
-
-    Returns:
-        (bmu_idx, second_bmu_idx, dist_bmu, dist_second)
-    """
-    grid_shape = weights.shape[:-1]
-    flat_w = weights.view(-1, 4)  # (I*J*K*L, 4)
-    diff = flat_w - x.view(1, 4)
-    dist_sq = torch.sum(diff * diff, dim=-1)  # (I*J*K*L,)
-
-    # Top-2 menores distâncias
-    top2 = torch.topk(dist_sq, k=2, largest=False)
-    flat_idx_bmu = int(top2.indices[0].item())
-    flat_idx_2nd = int(top2.indices[1].item())
-
-    # Unravel
-    L = grid_shape[3]
-    K = grid_shape[2]
-    J = grid_shape[1]
-
-    def unravel(flat_idx: int) -> Tuple[int, int, int, int]:
-        i = flat_idx // (J * K * L)
-        r = flat_idx % (J * K * L)
-        j = r // (K * L)
-        r = r % (K * L)
-        k = r // L
-        l = r % L
-        return (i, j, k, l)
-
-    bmu = unravel(flat_idx_bmu)
-    second = unravel(flat_idx_2nd)
-    dist_bmu = float(math.sqrt(max(top2.values[0].item(), 0.0)))
-    dist_second = float(math.sqrt(max(top2.values[1].item(), 0.0)))
-    return bmu, second, dist_bmu, dist_second
-
-
-# ============================================================================
-# Métricas principais (1.1 - 1.4)
-# ============================================================================
-def quantization_error(
-    data: torch.Tensor, weights: torch.Tensor
-) -> float:
-    """1.1 Erro de Quantização (QE).
-
-    QE = (1/N) * Σ_i ||x_i - W_BMU(x_i)||₂
-
-    Args:
-        data: tensor [N, 4] — vetores de entrada.
-        weights: tensor [I, J, K, L, 4] — pesos do SOM.
-
-    Returns:
-        QE (float). 0.0 se data vazio. NaN/Inf propagado é substituído por 0.0
-        (V6.5-V2-metrics-FIX-3: agora também captura Inf, não apenas NaN).
-    """
-    if data.numel() == 0:
-        return 0.0
-    flat_w = weights.view(-1, 4)  # (M, 4)
-    # distância de cada amostra a todos os neurônios
-    # data: (N, 4), flat_w: (M, 4) -> (N, M)
-    diff = data.unsqueeze(1) - flat_w.unsqueeze(0)
-    dist_sq = torch.sum(diff * diff, dim=-1)  # (N, M)
-    min_dist_sq, _ = torch.min(dist_sq, dim=1)  # (N,)
-    # V6.5-V2-metrics-FIX-2: clamp para evitar sqrt de número negativo (float err)
-    min_dist_sq = torch.clamp(min_dist_sq, min=0.0)
-    qe = torch.mean(torch.sqrt(min_dist_sq))
-    qe_val = float(qe.item())
-    # V6.5-V2-metrics-FIX-3: rejeita tanto NaN quanto Inf
-    if not math.isfinite(qe_val):
-        return 0.0
-    return qe_val
-
-
-def topological_error(
-    data: torch.Tensor, weights: torch.Tensor
-) -> float:
-    """1.2 Erro Topológico (TE).
-
-    TE = (1/N) * Σ_i 𝟙[¬adjacent(BMU_1(x_i), BMU_2(x_i))]
-
-    Dois neurônios são adjacentes se differem em exatamente uma coordenada
-    por ±1 (6-connectivity em 4D).
-
-    Args:
-        data: tensor [N, 4]
-        weights: tensor [I, J, K, L, 4]
-
-    Returns:
-        TE ∈ [0, 1]. 0.0 se data vazio ou se grid tem < 2 neurônios.
-    """
-    if data.numel() == 0:
-        return 0.0
-    # V6.5-V2-metrics-FIX-3: guard contra grid com < 2 neurônios
-    # (torch.topk(k=2) levanta RuntimeError se M < 2).
-    M = weights.shape[0] * weights.shape[1] * weights.shape[2] * weights.shape[3]
-    if M < 2:
-        return 0.0
-    grid_shape = weights.shape[:-1]
-    flat_w = weights.view(-1, 4)
-    diff = data.unsqueeze(1) - flat_w.unsqueeze(0)
-    dist_sq = torch.sum(diff * diff, dim=-1)  # (N, M)
-    # V6.5-V2-metrics-FIX-3: substitui NaN/Inf por +inf antes do topk
-    # para que neurônios com pesos degenerados não sejam eleitos BMU.
-    dist_sq = torch.where(torch.isfinite(dist_sq), dist_sq, torch.full_like(dist_sq, float('inf')))
-
-    # Top-2 BMUs para cada amostra
-    top2 = torch.topk(dist_sq, k=2, largest=False, dim=1)
-    flat_bmu = top2.indices[:, 0]  # (N,)
-    flat_2nd = top2.indices[:, 1]  # (N,)
-
-    # Unravel
-    L = grid_shape[3]
-    K = grid_shape[2]
-    J = grid_shape[1]
-    I = grid_shape[0]
-
-    def unravel_batch(flat_idx: torch.Tensor) -> torch.Tensor:
-        """Retorna tensor (N, 4) com (i, j, k, l)."""
-        i = flat_idx // (J * K * L)
-        r = flat_idx % (J * K * L)
-        j = r // (K * L)
-        r = r % (K * L)
-        k = r // L
-        l = r % L
-        return torch.stack([i, j, k, l], dim=1).long()
-
-    bmu_idx = unravel_batch(flat_bmu)  # (N, 4)
-    second_idx = unravel_batch(flat_2nd)  # (N, 4)
-
-    # Adjacência: |Δcoord| soma = 1 (apenas uma coordenada muda por ±1)
-    delta = (bmu_idx - second_idx).abs()  # (N, 4)
-    is_adjacent = (delta.sum(dim=1) == 1) & (delta.max(dim=1).values == 1)
-
-    te = 1.0 - float(is_adjacent.float().mean().item())
-    # V6.5-V2-metrics-FIX-3: rejeita NaN/Inf no resultado
-    if not math.isfinite(te):
-        return 0.0
-    return te
-
-
-def kaski_lagus_error(
-    data: torch.Tensor, weights: torch.Tensor, alpha: float = 0.5
-) -> float:
-    """1.3 Erro de Kaski-Lagus.
-
-    Combinação ponderada de QE normalizado e TE.
-
-    Variante implementada (Kaski & Lagus, 1999):
-        KL = α * MQE_norm + (1-α) * TE
-    onde MQE_norm = QE / mean(||x||) (normalização por escala dos dados).
-
-    Uma variante alternativa multiplica:
-        KL = QE * (1 + TE)
-    que penaliza QE quando TE é alto (usada se use_multiplicative=True).
-
-    Args:
-        data: tensor [N, 4]
-        weights: tensor [I, J, K, L, 4]
-        alpha: peso do QE normalizado (default 0.5)
-
-    Returns:
-        KL (float). 0.0 se data vazio ou se NaN for detectado
-        (V6.5-V2-metrics-FIX-2: user requirement "investigar QE e KL resultando
-        em NAN").
-    """
-    if data.numel() == 0:
-        return 0.0
-    qe = quantization_error(data, weights)
-    te = topological_error(data, weights)
-    # V6.5-V2-metrics-FIX-3: normalização canônica por desvio em torno do
-    # centróide (mean(||x_i - x̄||)), NÃO por mean(||x_i||). A versão antiga
-    # era sensível a translação dos dados — deslocar todos os vetores por +c
-    # inflava artificialmente ||x|| e reduzia KL, mesmo com QE/TE inalterados.
-    centroid = data.mean(dim=0, keepdim=True)  # (1, 4)
-    centered_norm = torch.norm(data - centroid, dim=1)  # (N,)
-    data_norm = float(centered_norm.mean().item())
-    # V6.5-V2-metrics-FIX-2: se data_norm é NaN ou zero, usa 1.0 como fallback
-    if not math.isfinite(data_norm) or data_norm < 1e-8:
-        data_norm = 1.0
-    qe_norm = qe / max(data_norm, 1e-8)
-    kl = alpha * qe_norm + (1.0 - alpha) * te
-    kl_val = float(kl)
-    # V6.5-V2-metrics-FIX-3: rejeita tanto NaN quanto Inf
-    if not math.isfinite(kl_val):
-        return 0.0
-    return kl_val
-
-
-def explained_variance_share(
-    data: torch.Tensor, weights: torch.Tensor
-) -> float:
-    """1.4 Variância Explicada (Share of Explained Variance).
-
-    VarExplained = 1 - Var(residual) / Var(total)
-    onde residual = x_i - W_BMU(x_i)
-
-    Implementação: computamos a variância total dos dados (soma das variâncias
-    por dimensão) e a variância dos resíduos. A fração explicada é:
-
-        VE = 1 - Σ_d Var(residual_d) / Σ_d Var(data_d)
-
-    Returns:
-        VE ∈ [0, 1]. Próximo de 1 = boa representação.
-    """
-    if data.numel() == 0 or data.shape[0] < 2:
-        return 0.0
-    flat_w = weights.view(-1, 4)
-    diff = data.unsqueeze(1) - flat_w.unsqueeze(0)
-    dist_sq = torch.sum(diff * diff, dim=-1)  # (N, M)
-    _, bmu_flat = torch.min(dist_sq, dim=1)  # (N,)
-    bmu_w = flat_w[bmu_flat]  # (N, 4)
-    residual = data - bmu_w  # (N, 4)
-
-    # Variância por dimensão (sem viés, ddof=0)
-    var_data = torch.var(data, dim=0, unbiased=False)  # (4,)
-    var_resid = torch.var(residual, dim=0, unbiased=False)  # (4,)
-
-    total_var = float(torch.sum(var_data).item())
-    resid_var = float(torch.sum(var_resid).item())
-    # V6.5-V2-metrics-FIX-3: se dados são constantes (var~0) e o SOM os
-    # representa perfeitamente (resid~0), a variância explicada é 1.0
-    # (caso degenerado, mas correto). Antes retornava 0.0 erroneamente.
-    if total_var < 1e-12:
-        return 1.0 if resid_var < 1e-12 else 0.0
-    ve = 1.0 - (resid_var / total_var)
-    ve_val = float(max(0.0, min(1.0, ve)))
-    # V6.5-V2-metrics-FIX-3: rejeita NaN/Inf
-    if not math.isfinite(ve_val):
-        return 0.0
-    return ve_val
-
-
-# ============================================================================
-# Indicadores de falha (2.1 - 2.4)
-# ============================================================================
-def topological_collapse(weights: torch.Tensor) -> Dict[str, Any]:
-    """2.1 Colapso Topológico (Topological Collapse).
-
-    Detecta se os pesos colapsaram para um ponto (rank < 1) ou para uma linha
-    (rank < 2). Usa SVD sobre os pesos reshaped (M, 4).
-
-    Args:
-        weights: tensor [I, J, K, L, 4]
-
-    Returns:
-        Dict com:
-          - collapsed_to_point: bool (todos iguais)
-          - collapsed_to_line: bool (variação só em 1 direção)
-          - effective_rank: float (rank efetivo via razão de singular values)
-          - singular_values: list[float]
-          - severity: str ("none" | "line" | "point")
-    """
-    flat_w = weights.view(-1, 4).clone()  # (M, 4)
-    if flat_w.numel() == 0:
-        return {"collapsed_to_point": False, "collapsed_to_line": False,
-                "effective_rank": 0.0, "singular_values": [], "severity": "none"}
-
-    # Centraliza
-    flat_w_centered = flat_w - flat_w.mean(dim=0, keepdim=True)
-
-    # V6.5-V2-metrics FIX: usa SVD com fallback robusto para matrizes
-    # singulares/ill-conditioned. Se SVD falhar, usa np.linalg.eigh com
-    # regularização (adiciona pequeno ruído diagonal).
-    S_list: List[float] = []
-    try:
-        # Tentativa 1: SVD direto (mais estável que eigh)
-        U, S, V = torch.linalg.svd(flat_w_centered, full_matrices=False)
-        S_list = [float(s.item()) for s in S]
-    except Exception:
-        # Tentativa 2: eigendecomposição da matriz de covariância com regularização
-        try:
-            cov = flat_w_centered.t() @ flat_w_centered
-            # Regularização: adiciona pequeno valor diagonal para estabilizar
-            n = cov.shape[0]
-            reg = 1e-6 * float(cov.diag().max().item()) if n > 0 else 1e-6
-            cov_reg = cov + torch.eye(n) * reg
-            S_eigh, _ = torch.linalg.eigh(cov_reg)
-            S_eigh = S_eigh.flip(0)  # descending order
-            # Converte autovalores em "singular values" aproximados (sqrt)
-            S_list = [float(math.sqrt(max(s.item(), 0.0))) for s in S_eigh]
-        except Exception:
-            # Tentativa 3: usa norma de Frobenius como aproximação
-            s_frob = float(torch.norm(flat_w_centered).item())
-            S_list = [s_frob, 0.0, 0.0, 0.0]
-
-    if not S_list:
-        return {"collapsed_to_point": False, "collapsed_to_line": False,
-                "effective_rank": 0.0, "singular_values": [], "severity": "none"}
-
-    # V6.5-V2-metrics-FIX-3: se SVD retornou NaN/Inf (pesos degenerados),
-    # marca explicitamente como corrupção por NaN para não mascarar o problema.
-    if any(not math.isfinite(s) for s in S_list):
-        return {
-            "collapsed_to_point": False, "collapsed_to_line": False,
-            "effective_rank": 0.0, "singular_values": S_list,
-            "severity": "nan_corrupt",
-            "error": "SVD returned NaN/Inf singular values (weights contain NaN/Inf)",
-        }
-
-    s_max = max(S_list[0] if S_list else 0.0, 1e-12)
-
-    # Effective rank: número de singular values > 1% do maior
-    threshold = 0.01 * s_max
-    n_significant = sum(1 for s in S_list if s > threshold)
-    # Effective rank (float, via entropy)
-    s_norm = [s / s_max for s in S_list]
-    s_norm = [s for s in s_norm if s > 1e-10]
-    if s_norm:
-        p = torch.tensor(s_norm)
-        p_norm = p / p.sum()
-        entropy = -float(torch.sum(p_norm * torch.log(p_norm)).item())
-        effective_rank = math.exp(entropy)
-    else:
-        effective_rank = 0.0
-
-    collapsed_to_point = n_significant <= 1 and s_max < 1e-6
-    collapsed_to_line = n_significant == 1 and s_max >= 1e-6
-
-    if collapsed_to_point:
-        severity = "point"
-    elif collapsed_to_line:
-        severity = "line"
-    else:
-        severity = "none"
-
-    return {
-        "collapsed_to_point": bool(collapsed_to_point),
-        "collapsed_to_line": bool(collapsed_to_line),
-        "effective_rank": float(effective_rank),
-        "singular_values": S_list,
-        "severity": severity,
-    }
-
-
-def dead_neuron_rate(
-    data: torch.Tensor, weights: torch.Tensor
-) -> Dict[str, Any]:
-    """2.2 Taxa de Neurônios Mortos (Dead Neurons).
-
-    DeadRate = (n_neurons - n_active_neurons) / n_neurons
-
-    Um neurônio é "ativo" se é BMU para pelo menos uma amostra.
-
-    Args:
-        data: tensor [N, 4]
-        weights: tensor [I, J, K, L, 4]
-
-    Returns:
-        Dict com:
-          - dead_neuron_rate: float ∈ [0, 1]
-          - n_dead: int
-          - n_active: int
-          - n_total: int
-          - dead_indices: list de (i,j,k,l)
-    """
-    n_total = weights.shape[0] * weights.shape[1] * weights.shape[2] * weights.shape[3]
-    if data.numel() == 0:
-        return {"dead_neuron_rate": 1.0, "n_dead": n_total, "n_active": 0,
-                "n_total": n_total, "dead_indices": []}
-
-    flat_w = weights.view(-1, 4)
-    diff = data.unsqueeze(1) - flat_w.unsqueeze(0)
-    dist_sq = torch.sum(diff * diff, dim=-1)  # (N, M)
-    # V6.5-V2-metrics-FIX-3: substitui NaN/Inf por +inf para que neurônios
-    # degenerados não sejam eleitos BMU e não causem IndexError.
-    dist_sq = torch.where(torch.isfinite(dist_sq), dist_sq, torch.full_like(dist_sq, float('inf')))
-    _, bmu_flat = torch.min(dist_sq, dim=1)  # (N,)
-    # V6.5-V2-metrics-FIX-3: valida range dos índices antes de indexar
-    bmu_flat = torch.clamp(bmu_flat, 0, n_total - 1)
-
-    active_mask = torch.zeros(n_total, dtype=torch.bool)
-    active_mask[bmu_flat] = True
-    n_active = int(active_mask.sum().item())
-    n_dead = n_total - n_active
-
-    # Coleta índices mortos
-    dead_flat = (~active_mask).nonzero(as_tuple=True)[0]
-    L_dim = weights.shape[3]
-    K_dim = weights.shape[2]
-    J_dim = weights.shape[1]
-    dead_indices = []
-    for flat_idx in dead_flat.tolist():
-        i = flat_idx // (J_dim * K_dim * L_dim)
-        r = flat_idx % (J_dim * K_dim * L_dim)
-        j = r // (K_dim * L_dim)
-        r = r % (K_dim * L_dim)
-        k = r // L_dim
-        l = r % L_dim
-        dead_indices.append((i, j, k, l))
-
-    return {
-        "dead_neuron_rate": float(n_dead / n_total) if n_total > 0 else 0.0,
-        "n_dead": int(n_dead),
-        "n_active": int(n_active),
-        "n_total": int(n_total),
-        "dead_indices": dead_indices[:50],  # limita para não explodir JSON
-    }
-
-
-def quantization_error_stagnation(
-    history: SOMMetricHistory,
-    high_threshold_frac: float = 0.7,
-    slope_tolerance: float = 1e-4,
-    window_size: int = 5,
-) -> Dict[str, Any]:
-    """2.3 Estagnação do Erro de Quantização.
-
-    Detecta se o QE estabilizou em patamar elevado:
-      - |slope(QE)| < slope_tolerance (estável)
-      - QE_atual > high_threshold_frac * max(QE_history) (alto)
-
-    Args:
-        history: histórico de métricas SOM.
-        high_threshold_frac: fração do QE máximo para considerar "alto".
-        slope_tolerance: tolerância de slope para considerar "estável".
-        window_size: nº de épocas recentes para regressão linear.
-
-    Returns:
-        Dict com:
-          - is_stagnant: bool
-          - current_qe: float
-          - max_qe: float
-          - slope: float
-          - severity: str ("none" | "mild" | "severe")
-    """
-    qe_list = list(history.qe_history)
-    if len(qe_list) < window_size + 2:
-        return {"is_stagnant": False, "current_qe": 0.0, "max_qe": 0.0,
-                "slope": 0.0, "severity": "none", "n_epochs": len(qe_list)}
-
-    # Regressão linear sobre as últimas window_size épocas
-    recent = qe_list[-window_size:]
-    n = len(recent)
-    x_mean = (n - 1) / 2.0
-    y_mean = sum(recent) / n
-    num = sum((i - x_mean) * (y - y_mean) for i, y in enumerate(recent))
-    den = sum((i - x_mean) ** 2 for i in range(n))
-    slope = num / den if den > 0 else 0.0
-
-    current_qe = float(qe_list[-1])
-    max_qe = float(max(qe_list))
-    # V6.5-V2-metrics-FIX-3: tolerância RELATIVA à magnitude do QE,
-    # não absoluta. Para QE~0.01, slope_tolerance=1e-4 representaria
-    # 1% por época (grande); para QE~100, seria negligenciável.
-    rel_tol = slope_tolerance * max(1.0, max_qe)
-    is_high = current_qe > high_threshold_frac * max_qe
-    is_stable = abs(slope) < rel_tol
-
-    is_stagnant = is_stable and is_high
-
-    if is_stagnant:
-        if current_qe > 0.9 * max_qe:
-            severity = "severe"
-        else:
-            severity = "mild"
-    else:
-        severity = "none"
-
-    return {
-        "is_stagnant": bool(is_stagnant),
-        "current_qe": current_qe,
-        "max_qe": max_qe,
-        "slope": float(slope),
-        "severity": severity,
-        "n_epochs": len(qe_list),
-    }
-
-
-def neighborhood_crossing(
-    history: SOMMetricHistory,
-    initial_window: int = 5,
-    final_window: int = 5,
-    growth_threshold: float = 1.5,
-) -> Dict[str, Any]:
-    """2.4 Cruzamento de Vizinhança (Neighborhood Crossing).
-
-    Detecta crescimento repentino ou oscilação crônica do TE nas épocas
-    finais:
-      - std(TE_final) > growth_threshold * std(TE_initial) (oscilação cresceu)
-      - TE[-1] > TE[-5] (crescimento no final)
-
-    Args:
-        history: histórico de métricas SOM.
-        initial_window: nº de épocas iniciais para baseline.
-        final_window: nº de épocas finais para comparação.
-        growth_threshold: razão de std para considerar "crescimento crônico".
-
-    Returns:
-        Dict com:
-          - detected: bool
-          - te_initial_mean: float
-          - te_final_mean: float
-          - te_initial_std: float
-          - te_final_std: float
-          - std_ratio: float
-          - is_oscillating: bool
-          - is_growing: bool
-          - severity: str ("none" | "mild" | "severe")
-    """
-    te_list = list(history.te_history)
-    if len(te_list) < initial_window + final_window:
-        return {"detected": False, "te_initial_mean": 0.0, "te_final_mean": 0.0,
-                "te_initial_std": 0.0, "te_final_std": 0.0, "std_ratio": 0.0,
-                "is_oscillating": False, "is_growing": False, "severity": "none",
-                "n_epochs": len(te_list)}
-
-    initial = te_list[:initial_window]
-    final = te_list[-final_window:]
-
-    # V6.5-V2-metrics-FIX-3: filtra NaN/Inf (defesa em profundidade — o
-    # SOMMetricHistory.record já deveria ter feito isso, mas chamadas
-    # diretas a neighborhood_crossing podem receber histórico corrupto).
-    initial = [v for v in initial if math.isfinite(v)]
-    final = [v for v in final if math.isfinite(v)]
-    if len(initial) < 2 or len(final) < 2:
-        return {"detected": False, "te_initial_mean": 0.0, "te_final_mean": 0.0,
-                "te_initial_std": 0.0, "te_final_std": 0.0, "std_ratio": 0.0,
-                "is_oscillating": False, "is_growing": False, "severity": "none",
-                "n_epochs": len(te_list)}
-
-    init_mean = sum(initial) / len(initial)
-    final_mean = sum(final) / len(final)
-    init_var = sum((x - init_mean) ** 2 for x in initial) / len(initial)
-    final_var = sum((x - final_mean) ** 2 for x in final) / len(final)
-    init_std = math.sqrt(init_var)
-    final_std = math.sqrt(final_var)
-
-    std_ratio = final_std / max(init_std, 1e-8)
-    is_oscillating = std_ratio > growth_threshold
-    is_growing = final_mean > init_mean and (final_mean - init_mean) > 0.05
-
-    detected = is_oscillating or is_growing
-
-    if detected:
-        if std_ratio > 2.0 * growth_threshold or is_growing:
-            severity = "severe"
-        else:
-            severity = "mild"
-    else:
-        severity = "none"
-
-    return {
-        "detected": bool(detected),
-        "te_initial_mean": float(init_mean),
-        "te_final_mean": float(final_mean),
-        "te_initial_std": float(init_std),
-        "te_final_std": float(final_std),
-        "std_ratio": float(std_ratio),
-        "is_oscillating": bool(is_oscillating),
-        "is_growing": bool(is_growing),
-        "severity": severity,
-        "n_epochs": len(te_list),
-    }
-
-
-# ============================================================================
-# Função agregadora: compute_all_metrics
-# ============================================================================
-def compute_all_metrics(
-    data: torch.Tensor,
-    weights: torch.Tensor,
-    history: Optional[SOMMetricHistory] = None,
-) -> Dict[str, Any]:
-    """Computa todas as métricas SOM de uma vez.
-
-    Args:
-        data: tensor [N, 4] — buffer de vetores 4D.
-        weights: tensor [I, J, K, L, 4] — pesos do SOM.
-        history: histórico de métricas (opcional, para estagnação e cruzamento).
-
-    Returns:
-        Dict com:
-          - quantization_error: float
-          - topological_error: float
-          - kaski_lagus_error: float
-          - explained_variance_share: float
-          - topological_collapse: Dict
-          - dead_neuron_rate: Dict
-          - quantization_error_stagnation: Dict (se history fornecido)
-          - neighborhood_crossing: Dict (se history fornecido)
-          - overall_health: str ("healthy" | "warning" | "critical")
-          - failure_indicators: list[str]
-    """
-    # V6.5-V2-metrics FIX: wrap entire computation in try/except para evitar
-    # crashes fatais quando o SOM tem pesos degenerados (ex: todos iguais
-    # após muito treinamento sem diversidade). Em caso de erro, retorna
-    # métricas safe-default em vez de crashar o treino.
-
-    # V6.5-V2-metrics-FIX-2 (user requirement: "investigar QE e KL resultando
-    # em NAN"): detecção EXPLÍCITA de NaN/Inf nos tensores de entrada ANTES
-    # de computar as métricas. Se os pesos do SOM contêm NaN (devido a
-    # gradientes explosivos durante train_hypotheses ou apply_best_delta),
-    # TODAS as métricas (QE, KL, etc.) seriam NaN, propagando o erro para
-    # o histórico e quebrando a detecção de estagnação/cruzamento.
-    # Esta barreira inicial retorna safe-defaults imediatamente quando
-    # NaN/Inf é detectado, evitando contaminação do histórico.
-    try:
-        if data is not None and data.numel() > 0:
-            if bool(torch.isnan(data).any().item()) or bool(torch.isinf(data).any().item()):
-                n_nan = int(torch.isnan(data).sum().item())
-                n_inf = int(torch.isinf(data).sum().item())
-                logger.warning(
-                    f"[som_metrics] NaN/Inf detected in INPUT DATA: "
-                    f"{n_nan} NaN, {n_inf} Inf out of {data.numel()} elements. "
-                    f"Returning safe-default metrics (skipping computation)."
-                )
-                return _nan_safe_metrics(
-                    reason=f"nan_in_data (nan={n_nan}, inf={n_inf})",
-                    data_shape=tuple(data.shape),
-                    weights_shape=tuple(weights.shape) if weights is not None else None,
-                )
-        if weights is not None and weights.numel() > 0:
-            if bool(torch.isnan(weights).any().item()) or bool(torch.isinf(weights).any().item()):
-                n_nan = int(torch.isnan(weights).sum().item())
-                n_inf = int(torch.isinf(weights).sum().item())
-                logger.warning(
-                    f"[som_metrics] NaN/Inf detected in SOM WEIGHTS: "
-                    f"{n_nan} NaN, {n_inf} Inf out of {weights.numel()} elements. "
-                    f"Returning safe-default metrics (skipping computation). "
-                    f"This usually indicates exploding gradients in "
-                    f"train_hypotheses() or apply_best_delta_and_consolidate()."
-                )
-                return _nan_safe_metrics(
-                    reason=f"nan_in_weights (nan={n_nan}, inf={n_inf})",
-                    data_shape=tuple(data.shape) if data is not None else None,
-                    weights_shape=tuple(weights.shape),
-                )
-    except Exception as nan_check_err:
-        logger.warning(
-            f"[som_metrics] NaN pre-check failed (proceeding to compute): {nan_check_err}"
-        )
-
-    try:
-        result = _compute_all_metrics_impl(data, weights, history)
-        # V6.5-V2-metrics-FIX-3: pós-computação, verifica TODAS as métricas
-        # escalares por NaN/Inf (não apenas QE/KL). Substitui por 0.0 e marca
-        # para que o histórico não seja contaminado.
-        _SCALAR_METRIC_KEYS = [
-            "quantization_error", "topological_error", "kaski_lagus_error",
-            "explained_variance_share",
-        ]
-        any_nan = False
-        for k in _SCALAR_METRIC_KEYS:
-            v = result.get(k, 0.0)
-            if isinstance(v, float) and not math.isfinite(v):
-                logger.warning(
-                    f"[som_metrics] {k} computed as NaN/Inf ({v}) — replacing "
-                    f"with 0.0. Investigate train_hypotheses or apply_best_delta "
-                    f"for gradient explosion."
-                )
-                result[k] = 0.0
-                result[f"{k}_was_nan"] = True
-                any_nan = True
-        # V6.5-V2-metrics-FIX-3: também verifica sub-dicts de collapse/dead
-        # para severity='nan_corrupt' e propaga como failure_indicator.
-        if result.get("topological_collapse", {}).get("severity") == "nan_corrupt":
-            any_nan = True
-        if any_nan and "failure_indicators" in result:
-            result["failure_indicators"].append("post_compute_nan_replaced")
-            result["overall_health"] = "critical"
-            result["n_failure_indicators"] = len(result["failure_indicators"])
-        return result
-    except Exception as e:
-        logger.warning(
-            f"[som_metrics] compute_all_metrics failed (using safe defaults): {e}"
-        )
-        return _nan_safe_metrics(reason=f"compute_error: {str(e)[:200]}", data_shape=None, weights_shape=None)
-
-
-def _nan_safe_metrics(
-    reason: str,
-    data_shape: Optional[Tuple[int, ...]] = None,
-    weights_shape: Optional[Tuple[int, ...]] = None,
-) -> Dict[str, Any]:
-    """V6.5-V2-metrics-FIX-2 — Retorna métricas safe-default quando NaN é detectado.
-
-    User requirement: "Aprimorar a FASE2 PUNITIVA investigar QE e KL resultando em NAN".
-
-    Este helper centraliza a resposta a NaN/Inf em qualquer ponto do pipeline
-    de métricas, garantindo que:
-      1. O treino NÃO crashe.
-      2. O histórico de métricas (SOMMetricHistory) NÃO seja contaminado com NaN
-         (o que quebraria detecção de estagnação e cruzamento de vizinhança).
-      3. O log indique claramente a fonte do NaN (data vs weights vs compute_error).
-      4. failure_indicators registre o evento para análise pós-treino.
-    """
-    return {
-        "quantization_error": 0.0,
-        "topological_error": 0.0,
-        "kaski_lagus_error": 0.0,
-        "explained_variance_share": 0.0,
-        "topological_collapse": {
-            "collapsed_to_point": False, "collapsed_to_line": False,
-            "effective_rank": 0.0, "singular_values": [],
-            "severity": "none", "error": reason,
-        },
-        "dead_neuron_rate": {
-            "dead_neuron_rate": 1.0, "n_dead": 0, "n_active": 0,
-            "n_total": 0, "dead_indices": [], "error": reason,
-        },
-        "qe_stagnation": {
-            "is_stagnant": False, "current_qe": 0.0, "max_qe": 0.0,
-            "slope": 0.0, "severity": "none", "n_epochs": 0,
-        },
-        "neighborhood_crossing": {
-            "detected": False, "te_initial_mean": 0.0, "te_final_mean": 0.0,
-            "te_initial_std": 0.0, "te_final_std": 0.0, "std_ratio": 0.0,
-            "is_oscillating": False, "is_growing": False, "severity": "none",
-            "n_epochs": 0,
-        },
-        "overall_health": "critical",
-        "failure_indicators": [f"nan_detected: {reason[:120]}"],
-        "n_failure_indicators": 1,
-        "nan_detected": True,
-        "nan_reason": reason,
-        "data_shape": list(data_shape) if data_shape else None,
-        "weights_shape": list(weights_shape) if weights_shape else None,
-    }
-
-
-def _compute_all_metrics_impl(
-    data: torch.Tensor,
-    weights: torch.Tensor,
-    history: Optional[SOMMetricHistory] = None,
-) -> Dict[str, Any]:
-    """Implementação interna de compute_all_metrics (sem try/except wrapper)."""
-    metrics: Dict[str, Any] = {}
-
-    # 1.1 - 1.4: Métricas principais
-    metrics["quantization_error"] = quantization_error(data, weights)
-    metrics["topological_error"] = topological_error(data, weights)
-    metrics["kaski_lagus_error"] = kaski_lagus_error(data, weights)
-    metrics["explained_variance_share"] = explained_variance_share(data, weights)
-
-    # 2.1: Colapso topológico
-    collapse = topological_collapse(weights)
-    metrics["topological_collapse"] = collapse
-
-    # 2.2: Dead neurons
-    dead = dead_neuron_rate(data, weights)
-    metrics["dead_neuron_rate"] = dead
-
-    # 2.3: Estagnação do QE (requer histórico)
-    if history is not None and len(history.qe_history) > 5:
-        metrics["qe_stagnation"] = quantization_error_stagnation(history)
-    else:
-        metrics["qe_stagnation"] = {
-            "is_stagnant": False, "current_qe": metrics["quantization_error"],
-            "max_qe": metrics["quantization_error"], "slope": 0.0,
-            "severity": "none", "n_epochs": 0,
-        }
-
-    # 2.4: Cruzamento de vizinhança (requer histórico)
-    if history is not None and len(history.te_history) > 10:
-        metrics["neighborhood_crossing"] = neighborhood_crossing(history)
-    else:
-        metrics["neighborhood_crossing"] = {
-            "detected": False, "te_initial_mean": 0.0, "te_final_mean": 0.0,
-            "te_initial_std": 0.0, "te_final_std": 0.0, "std_ratio": 0.0,
-            "is_oscillating": False, "is_growing": False, "severity": "none",
-            "n_epochs": 0,
-        }
-
-    # Avaliação de saúde geral
-    failure_indicators: List[str] = []
-    if collapse["collapsed_to_point"]:
-        failure_indicators.append("topological_collapse_point")
-    elif collapse["collapsed_to_line"]:
-        failure_indicators.append("topological_collapse_line")
-    if dead["dead_neuron_rate"] > 0.5:
-        failure_indicators.append(f"dead_neuron_rate_high_{dead['dead_neuron_rate']:.2f}")
-    if metrics["qe_stagnation"]["is_stagnant"]:
-        failure_indicators.append(f"qe_stagnant_{metrics['qe_stagnation']['severity']}")
-    if metrics["neighborhood_crossing"]["detected"]:
-        failure_indicators.append(
-            f"neighborhood_crossing_{metrics['neighborhood_crossing']['severity']}"
-        )
-    if metrics["explained_variance_share"] < 0.3:
-        failure_indicators.append(
-            f"low_explained_variance_{metrics['explained_variance_share']:.2f}"
-        )
-    if metrics["topological_error"] > 0.5:
-        failure_indicators.append(
-            f"high_topological_error_{metrics['topological_error']:.2f}"
-        )
-
-    if len(failure_indicators) >= 2:
-        overall_health = "critical"
-    elif len(failure_indicators) == 1:
-        overall_health = "warning"
-    else:
-        overall_health = "healthy"
-
-    metrics["failure_indicators"] = failure_indicators
-    metrics["overall_health"] = overall_health
-    metrics["n_failure_indicators"] = len(failure_indicators)
-
-    return metrics
-
-
-__all__ = [
-    "SOMMetricHistory",
-    "quantization_error",
-    "topological_error",
-    "kaski_lagus_error",
-    "explained_variance_share",
-    "topological_collapse",
-    "dead_neuron_rate",
-    "quantization_error_stagnation",
-    "neighborhood_crossing",
-    "compute_all_metrics",
-]

streaming_datasets.py

@@ -1,995 +0,0 @@
-"""streaming_datasets_v13_9.py — Streaming dataset loader for v13.9.1 training.
-
-Carrega os 9 datasets especificados pelo usuário via streaming (IterableDataset),
-um por vez, exaustivamente dentro do limite max_samples_per_dataset.
-
-Datasets:
-  1. CEIA-POSITIVO/ultrachat_br_clustred_balanced_v1
-  2. Madras1/corpus-ptbr-v2
-  3. rhaymison/multmodal_175k_portuguese
-  4. TucanoBR/GigaVerbo
-  5. nvidia/OpenMathReasoning
-  6. MathLLMs/MathVision
-  7. nvidia/OpenMathInstruct-2
-  8. dominguesm/restore-punctuation-ptbr-dataset
-  9. carolina-c4ai/corpus-carolina
-
-Memory-efficient: streaming=True, no full materialization.
-Returns ProcessedSample with raw_text only (tokenizer handles encoding).
-"""
-from __future__ import annotations
-
-import logging
-import os
-import time
-import traceback
-from dataclasses import dataclass, field
-from typing import Any, Dict, Iterator, List, Optional, Tuple
-
-import torch
-
-logger = logging.getLogger(__name__)
-
-# Datasets V13.9.1 — exatamente os 9 especificados pelo usuário
-# V6.5-final: adicionado dominguesm/Canarim-Instruct-PTBR-Dataset
-# V6.5-7ds: adicionados adalbertojunior/punctuation-ptbr,
-#           iara-project/news-articles-ptbr-dataset, manoela/noticias_ptbr
-# V6.5-V2: adicionado BrunoN-Dev/corpus-ptbr-v1 como 8º dataset de CONHECIMENTO
-#          e usado como dataset de TREINAMENTO COM PUNIÇÃO
-DEFAULT_DATASETS = [
-    "CEIA-POSITIVO/ultrachat_br_clustred_balanced_v1",
-    "Madras1/corpus-ptbr-v2",
-    "rhaymison/multmodal_175k_portuguese",
-    "TucanoBR/GigaVerbo",
-    "nvidia/OpenMathReasoning",
-    "MathLLMs/MathVision",
-    "nvidia/OpenMathInstruct-2",
-    "dominguesm/restore-punctuation-ptbr-dataset",
-    "carolina-c4ai/corpus-carolina",
-    "dominguesm/Canarim-Instruct-PTBR-Dataset",
-    "adalbertojunior/punctuation-ptbr",
-    "iara-project/news-articles-ptbr-dataset",
-    "manoela/noticias_ptbr",
-    "BrunoN-Dev/corpus-ptbr-v1",
-]
-
-# V6.5-V2 — Sequência de 8 datasets para a fase CONHECIMENTO (na ordem exata
-# especificada pelo usuário). Meta mínima: 8000 samples ou mais.
-CONHECIMENTO_DATASETS_V2 = [
-    "dominguesm/restore-punctuation-ptbr-dataset",
-    "carolina-c4ai/corpus-carolina",
-    "CEIA-POSITIVO/ultrachat_br_clustred_balanced_v1",
-    "dominguesm/Canarim-Instruct-PTBR-Dataset",
-    "adalbertojunior/punctuation-ptbr",
-    "iara-project/news-articles-ptbr-dataset",
-    "manoela/noticias_ptbr",
-    "BrunoN-Dev/corpus-ptbr-v1",
-]
-
-# V6.5-V2 — Dataset para TREINAMENTO COM PUNIÇÃO (500 samples).
-# O usuário especificou: "dataset para TREINAMENTO COM PUNIÇÃO ATIVA
-# 'BrunoN-Dev/corpus-ptbr-v1'".
-PUNICAO_DATASET_V2 = "BrunoN-Dev/corpus-ptbr-v1"
-
-# Dataset format descriptors — Lista priorizada de campos de texto
-DATASET_FORMATS: Dict[str, Dict[str, Any]] = {
-    "CEIA-POSITIVO/ultrachat_br_clustred_balanced_v1": {
-        "type": "chat",
-        "text_fields": ["conversa", "text", "content", "conversation", "messages"],
-        "label_fields": [],
-        "split": "train",
-        "config": None,
-    },
-    "Madras1/corpus-ptbr-v2": {
-        "type": "text",
-        "text_fields": ["text", "content", "document", "body"],
-        "label_fields": [],
-        "split": "train",
-        "config": None,
-    },
-    "rhaymison/multmodal_175k_portuguese": {
-        "type": "multimodal",
-        "text_fields": ["description", "text", "question", "prompt", "instruction", "input"],
-        "image_fields": ["image", "image_url"],
-        "label_fields": ["answer", "response", "output"],
-        "split": "train",
-        "config": None,
-    },
-    "TucanoBR/GigaVerbo": {
-        "type": "text",
-        "text_fields": ["text", "content", "document", "body"],
-        "label_fields": [],
-        "split": "train",
-        "config": None,
-    },
-    "nvidia/OpenMathReasoning": {
-        "type": "math",
-        "text_fields": ["problem", "question", "input", "prompt"],
-        "label_fields": ["solution", "answer", "output", "response"],
-        # BUG FIX: OpenMathReasoning has splits ['cot', 'tir', 'genselect', 'additional_problems']
-        "split": "cot",  # Use 'cot' (chain-of-thought) as primary split
-        "config": None,
-    },
-    "MathLLMs/MathVision": {
-        "type": "math_multimodal",
-        "text_fields": ["question", "problem", "text", "query"],
-        "image_fields": ["image", "image_url"],
-        "label_fields": ["answer", "solution", "response"],
-        # BUG FIX: MathVision has only 'test' and 'testmini' splits (no train)
-        "split": "test",
-        "config": None,
-    },
-    "nvidia/OpenMathInstruct-2": {
-        "type": "math",
-        "text_fields": ["problem", "question", "input", "prompt"],
-        # V3 FIX: campos reais do dataset são generated_solution e expected_answer
-        "label_fields": ["generated_solution", "expected_answer", "solution", "answer", "output", "response"],
-        "split": "train",
-        "config": None,
-    },
-    "dominguesm/restore-punctuation-ptbr-dataset": {
-        "type": "punctuation",
-        "text_fields": ["text", "original", "unpunctuated", "input"],
-        "label_fields": ["punctuated", "restored", "target", "output"],
-        "split": "train",
-        "config": None,
-    },
-    "carolina-c4ai/corpus-carolina": {
-        "type": "text",
-        "text_fields": ["text", "content", "document", "body", "xml"],
-        "label_fields": [],
-        "split": "train",
-        "config": None,
-        # V13.9.2-carolina: dataset original usa script Python (não suportado em
-        # datasets 5.0+). Workaround: stream_xml_gz direto via lxml.iterparse,
-        # sem usar o script Python. Implementado em _stream_carolina_direct().
-    },
-    # ── V13.9.2-finetune-v2: novos datasets PT-BR ───────────────────────────
-    # Adicionados para fine-tuning contínuo do modelo v13.9.2 (preserva
-    # arquitetura e parâmetros). Nenhuma mudança estrutural — apenas novos
-    # descritores de formato para que stream_dataset() reconheça os datasets.
-    #
-    # V13.9.2-finetune-v3: padronização para o formato unificado
-    # "### Instruction:\n...\n\n### Response:\n..." conforme requisitado pelo
-    # usuário. Apenas os datasets com estrutura instruction/response (orion,
-    # cnmoro) usam o template; strak2005/bratao continuam como texto puro
-    # (corpus sem estrutura de instrução). Nenhuma mudança nos 9 datasets
-    # base — retrocompatibilidade total.
-    "orion-research/translations-en_US-pt_BR": {
-        "type": "translation",
-        # string = EN (instruction), string_translation = PT-BR (response).
-        # V13.9.2-finetune-v3: separa instruction/response para o template
-        # unificado em vez de concatenar como texto único.
-        "text_fields": ["string"],
-        "label_fields": ["string_translation"],
-        "split": "train",
-        "config": None,
-        "format_template": "instruction_response",
-        "instruction_prefix": "Translate the following text to Portuguese:",
-    },
-    "cnmoro/Instruct-PTBR-10M": {
-        "type": "instruct",
-        # NB: campos em MAIÚSCULAS no schema do dataset
-        # V13.9.2-finetune-v3: INSTRUCTION -> instruction, RESPONSE -> response
-        # via template unificado.
-        "text_fields": ["INSTRUCTION"],
-        "label_fields": ["RESPONSE"],
-        "split": "train",
-        "config": None,
-        "format_template": "instruction_response",
-        # V13.9.2-finetune-v2: dataset é um único parquet de 10 GB — streaming
-        # via datasets.load_dataset() é extremamente lento para materializar
-        # a primeira amostra. Fallback via HF datasets-server rows API (HTTP),
-        # que retorna batches de 100 rows rapidamente. Implementado em
-        # _stream_via_rows_api().
-        "loader": "rows_api",
-    },
-    "strak2005/corpus-ptbr-v1": {
-        "type": "text",
-        "text_fields": ["text", "content"],
-        "label_fields": [],
-        "split": "train",
-        "config": None,
-        # Fallback: se strak2005 cair, tenta bratao (mirror idêntico)
-        "fallback": "bratao/corpus-ptbr-v1",
-        # V13.9.2-finetune-v2: dataset tem 18 shards parquet de ~1GB cada.
-        # Streaming via datasets.load_dataset() não materializa primeira
-        # amostra em tempo aceitável. Fallback: baixa 1 shard localmente e
-        # itera com pyarrow. Implementado em _stream_via_parquet_first_shard().
-        "loader": "parquet_first_shard",
-    },
-    "bratao/corpus-ptbr-v1": {
-        "type": "text",
-        "text_fields": ["text", "content"],
-        "label_fields": [],
-        "split": "train",
-        "config": None,
-        "loader": "parquet_first_shard",
-    },
-    # ── V6.5: novos datasets para esgotar ────────────────────────────────
-    # User requirement: "esgotar 'dominguesm/restore-punctuation-ptbr-dataset'
-    # e 'carolina-c4ai/corpus-carolina' e 'nvidia/OpenMathInstruct-2' e
-    # 'nvidia/OpenMathReasoning' e 'CEIA-POSITIVO/ultrachat_br_clustred_balanced_v1'
-    # e 'Dexavator/English-PTBR'"
-    #
-    # Dexavator/English-PTBR: dataset de tradução EN->PT-BR.
-    # Estrutura típica: {"english": "...", "portuguese": "..."} ou
-    # {"en": "...", "pt": "..."}.
-    "Dexavator/English-PTBR": {
-        "type": "translation",
-        "text_fields": ["english", "en", "text", "source"],
-        "label_fields": ["portuguese", "pt", "target", "translation"],
-        "split": "train",
-        "config": None,
-        "format_template": "instruction_response",
-        "instruction_prefix": "Translate the following text to Portuguese:",
-    },
-    # ── V6.5-final: Canarim-Instruct-PTBR-Dataset ────────────────────────
-    # User requirement: "esgotar 'dominguesm/Canarim-Instruct-PTBR-Dataset'"
-    # Canarim-Instruct: dataset PT-BR de instruções (~430k samples).
-    # Estrutura típica: {"instruction": "...", "input": "...", "output": "..."}
-    # ou {"text": "...", "conversation": [{"role": "user", "content": "..."}, ...]}
-    "dominguesm/Canarim-Instruct-PTBR-Dataset": {
-        "type": "instruct",
-        "text_fields": ["instruction", "input", "text", "prompt", "question"],
-        "label_fields": ["output", "response", "answer"],
-        "split": "train",
-        "config": None,
-        "format_template": "instruction_response",
-        "instruction_prefix": "Instrução:",
-    },
-    # ── V6.5-7ds: novos 3 datasets PT-BR ─────────────────────────────────
-    # User requirement (latest): "streaming até esgotar nesta sequência
-    # 'dominguesm/restore-punctuation-ptbr-dataset' e
-    # 'carolina-c4ai/corpus-carolina' e
-    # 'CEIA-POSITIVO/ultrachat_br_clustred_balanced_v1' e
-    # 'dominguesm/Canarim-Instruct-PTBR-Dataset' e
-    # 'adalbertojunior/punctuation-ptbr' e
-    # 'iara-project/news-articles-ptbr-dataset' e
-    # 'manoela/noticias_ptbr'"
-    #
-    # adalbertojunior/punctuation-ptbr:
-    #   Dataset de restauração de pontuação PT-BR. Schema:
-    #     {id: str, tokens: list[str], pos_tags, chunk_tags, ner_tags}
-    #   O campo "tokens" é uma lista de palavras (sem pontuação). Precisamos
-    #   fazer join com espaços para obter texto bruto. Como o dataset usa
-    #   script Python (datasets 5.0+ não suporta), usamos rows_api loader.
-    #   Splits: train/validation/test com config "punctuation-ptbr".
-    "adalbertojunior/punctuation-ptbr": {
-        "type": "punctuation_tokens",
-        "text_fields": ["tokens"],  # list[str] — join com espaços
-        "label_fields": [],
-        "split": "train",
-        "config": "punctuation-ptbr",  # necessário para rows_api
-        "loader": "rows_api",
-        "join_tokens": True,  # flag especial para _normalize_sample
-    },
-    # iara-project/news-articles-ptbr-dataset:
-    #   Dataset de notícias PT-BR da Folha de S.Paulo (via IARA project).
-    #   Schema: {title, text, date, category, category_natural_language, link}
-    #   Streaming normal via parquet — primeira amostra materializa rápido.
-    #   "text" é o corpo da notícia, "title" é o título. Concatenamos ambos
-    #   via format_template="title_text_concat" para enriquecer contexto.
-    #   O campo "category" é usado como label (categoria da notícia).
-    "iara-project/news-articles-ptbr-dataset": {
-        "type": "news",
-        # text_fields em ordem de prioridade: "text" (corpo) primeiro
-        "text_fields": ["text", "title", "content", "body"],
-        "label_fields": ["category", "category_natural_language", "title"],
-        "split": "train",
-        "config": None,
-        # V6.5-7ds: title_text_concat usa title como label (prefixo)
-        # e text como corpo. Aqui label_fields pega "category" primeiro
-        # para dar contexto temático.
-        "format_template": "title_text_concat",
-    },
-    # manoela/noticias_ptbr:
-    #   Mirror do iara-project/news-articles-ptbr-dataset (mesmo parquet,
-    #   mesmo schema). Streaming normal via parquet.
-    "manoela/noticias_ptbr": {
-        "type": "news",
-        "text_fields": ["text", "title", "content", "body"],
-        "label_fields": ["category", "category_natural_language", "title"],
-        "split": "train",
-        "config": None,
-        "format_template": "title_text_concat",
-    },
-    # ── V6.5-V2: BrunoN-Dev/corpus-ptbr-v1 ───────────────────────────────
-    # User requirement: "dataset para TREINAMENTO COM PUNIÇÃO ATIVA
-    # 'BrunoN-Dev/corpus-ptbr-v1'" e o 8º dataset de CONHECIMENTO.
-    #
-    # BrunoN-Dev/corpus-ptbr-v1: corpus PT-BR genérico com 18 shards parquet
-    # (~1GB cada). Schema: {"text": "...", "meta": "..."}.
-    # Streaming via datasets.load_dataset(streaming=True) é lento para
-    # materializar a primeira amostra — usamos o loader parquet_first_shard
-    # (baixa apenas o primeiro shard e itera localmente com pyarrow).
-    # Sem label explícita — usamos hash do texto (paridade do nº de palavras)
-    # para gerar label binário (0/1) — não é dado sintético, é derivação
-    # algorítmica do conteúdo real do dataset.
-    "BrunoN-Dev/corpus-ptbr-v1": {
-        "type": "text",
-        "text_fields": ["text", "content", "body", "document"],
-        "label_fields": [],
-        "split": "train",
-        "config": None,
-        "loader": "parquet_first_shard",
-        # V6.5-V2: gera label binário determinístico via hash do texto
-        # (paridade do número de palavras) — não é dado sintético, é
-        # derivação algorítmica do conteúdo real do dataset.
-        "label_strategy": "hash_parity",
-    },
-}
-
-
-# Lista de arquivos XML.gz do Carolina que tentaremos baixar (ordem: menores
-# taxonomias primeiro para minimizar banda e RAM). Selecionamos 2 arquivos
-# por taxonomia para garantir diversidade sem explodir a memória.
-CAROLINA_XML_FILES = [
-    # taxonomia "datasets_and_other_corpora" (DAT = pt-BR notícias)
-    "corpus/datasets_and_other_corpora/pt-BR/DATa.xml.gz",
-    "corpus/datasets_and_other_corpora/pt-BR/DATb.xml.gz",
-    # taxonomia "wik" (wikis)
-    "corpus/wik/WIKa.xml.gz",
-    "corpus/wik/WIKb.xml.gz",
-    # taxonomia "uni" (university_domains)
-    "corpus/uni/UNIa.xml.gz",
-    "corpus/uni/UNIb.xml.gz",
-    # taxonomia "pub" (public_domain_works)
-    "corpus/pub/PUBa.xml.gz",
-    "corpus/pub/PUBb.xml.gz",
-]
-
-
-def _stream_carolina_direct(
-    max_samples: int,
-    hf_token: Optional[str] = None,
-    timeout_per_file: int = 60,
-) -> Iterator[Dict[str, Any]]:
-    """Workaround V13.9.2-carolina: carrega Carolina sem usar o script Python.
-
-    Baixa arquivos XML.gz diretamente do repo e faz parse streaming com
-    lxml.etree.iterparse (compatível com datasets 5.0+). Cada <TEI> vira um
-    documento de texto (concatenação dos <p> dentro de <body>).
-
-    Args:
-        max_samples: número máximo de amostras a retornar
-        hf_token: token HF opcional (dataset é público, mas token evita rate limit)
-        timeout_per_file: tempo máximo (segundos) por arquivo XML
-
-    Yields:
-        dict com chave "text" (compatível com _normalize_sample)
-    """
-    from huggingface_hub import hf_hub_download
-    import gzip
-    from lxml import etree
-
-    TEI_NS = "{http://www.tei-c.org/ns/1.0}"
-    BODY_P_TAG = f".//{TEI_NS}body/{TEI_NS}p"
-
-    count = 0
-    for xml_path in CAROLINA_XML_FILES:
-        if count >= max_samples:
-            return
-        try:
-            local_path = hf_hub_download(
-                repo_id="carolina-c4ai/corpus-carolina",
-                filename=xml_path,
-                repo_type="dataset",
-                token=hf_token,
-            )
-        except Exception as e:
-            logger.warning(f"Carolina: não foi possível baixar {xml_path}: {e}")
-            continue
-
-        try:
-            with gzip.open(local_path, "rb") as gz:
-                # iterparse streaming: não carrega árvore inteira na memória
-                for _, tei in etree.iterparse(
-                    gz, huge_tree=True, encoding="utf-8", tag=f"{TEI_NS}TEI"
-                ):
-                    if count >= max_samples:
-                        tei.clear()
-                        return
-                    # Extrai texto dos <p> dentro de <body>
-                    parts = []
-                    for p in tei.findall(BODY_P_TAG):
-                        if p.text:
-                            parts.append(p.text)
-                    text = " ".join(parts).strip()
-                    if text and len(text) >= 10:
-                        yield {"text": text, "meta": ""}
-                    tei.clear()  # libera memória da árvore TEI
-                    count += 1
-        except Exception as e:
-            logger.warning(f"Carolina: erro processando {xml_path}: {e}")
-            continue
-
-    logger.info(f"Carolina: streamou {count} amostras no total")
-
-
-# ═══════════════════════════════════════════════════════════════════════════
-# V13.9.2-finetune-v2: Loaders alternativos para datasets parquet grandes
-# onde datasets.load_dataset(streaming=True) é inviável (primeira amostra
-# demora minutos para materializar).
-# ═══════════════════════════════════════════════════════════════════════════
-
-def _stream_via_rows_api(
-    dataset_name: str,
-    max_samples: int,
-    hf_token: Optional[str] = None,
-    batch_size: int = 100,
-    max_offset: int = 100_000,
-    config: Optional[str] = None,
-    split: str = "train",
-) -> Iterator[Dict[str, Any]]:
-    """Carrega amostras via HF datasets-server /rows API (HTTP).
-
-    Usado para datasets cujo parquet é muito grande para streaming eficiente
-    (ex: cnmoro/Instruct-PTBR-10M com 10 GB em arquivo único) OU datasets
-    que usam script Python como loader (não suportado em datasets 5.0+,
-    ex: adalbertojunior/punctuation-ptbr). A rows API retorna batches de
-    até 100 rows via HTTP, sem precisar baixar o parquet.
-
-    Args:
-        dataset_name: repo_id do dataset (ex: "cnmoro/Instruct-PTBR-10M")
-        max_samples: nº máximo de amostras a retornar
-        hf_token: token HF (opcional — dataset público)
-        batch_size: nº de rows por requisição (máx 100)
-        max_offset: offset máximo a tentar (sai do loop se excedido)
-        config: nome da config do dataset (default: "default"). NECESSÁRIO
-                para datasets cuja config não é "default" (ex:
-                adalbertojunior/punctuation-ptbr usa "punctuation-ptbr").
-        split: nome do split (default: "train").
-
-    Yields:
-        dict com campos do dataset (compatível com _normalize_sample)
-    """
-    import json as _json
-    import urllib.request
-    import urllib.parse
-
-    base_url = "https://datasets-server.huggingface.co/rows"
-    config_name = config or "default"
-    count = 0
-    offset = 0
-
-    while count < max_samples and offset < max_offset:
-        params = urllib.parse.urlencode({
-            "dataset": dataset_name,
-            "config": config_name,
-            "split": split,
-            "offset": offset,
-            "length": min(batch_size, max_samples - count),
-        })
-        url = f"{base_url}?{params}"
-        try:
-            req = urllib.request.Request(url)
-            if hf_token:
-                req.add_header("Authorization", f"Bearer {hf_token}")
-            with urllib.request.urlopen(req, timeout=30) as r:
-                data = _json.loads(r.read())
-        except Exception as e:
-            logger.warning(f"rows_api {dataset_name} config={config_name} offset={offset}: {e}")
-            break
-
-        rows = data.get("rows", [])
-        if not rows:
-            break
-
-        for row in rows:
-            if count >= max_samples:
-                return
-            row_data = row.get("row", {})
-            # Remove campos internos como __index_level_0__
-            clean = {k: v for k, v in row_data.items()
-                     if not k.startswith("__")}
-            yield clean
-            count += 1
-
-        offset += len(rows)
-        if len(rows) < batch_size:
-            break  # chegou ao fim do dataset
-
-    logger.info(f"rows_api {dataset_name} config={config_name}: streamou {count} amostras")
-
-
-def _stream_via_parquet_first_shard(
-    dataset_name: str,
-    max_samples: int,
-    hf_token: Optional[str] = None,
-) -> Iterator[Dict[str, Any]]:
-    """Baixa primeiro shard parquet do dataset e itera localmente com pyarrow.
-
-    Usado para datasets com múltiplos shards parquet grandes (ex:
-    strak2005/corpus-ptbr-v1 com 18 shards de ~1GB). Streaming via
-    datasets.load_dataset() não materializa primeira amostra em tempo
-    aceitável. Esta função baixa o primeiro shard (~26s para 1GB) e itera
-    localmente — rápido após download.
-
-    Args:
-        dataset_name: repo_id do dataset
-        max_samples: nº máximo de amostras
-        hf_token: token HF opcional
-
-    Yields:
-        dict com campos do dataset
-    """
-    from huggingface_hub import HfApi, hf_hub_download
-    import pyarrow.parquet as pq
-
-    api = HfApi(token=hf_token)
-
-    # Lista arquivos parquet no repo
-    try:
-        info = api.dataset_info(dataset_name)
-    except Exception as e:
-        logger.warning(f"parquet_shard {dataset_name}: dataset_info failed: {e}")
-        return
-
-    parquet_files = sorted([
-        s.rfilename for s in info.siblings
-        if s.rfilename.endswith(".parquet")
-    ])
-    if not parquet_files:
-        logger.warning(f"parquet_shard {dataset_name}: nenhum .parquet encontrado")
-        return
-
-    # Baixa apenas o primeiro shard
-    first_shard = parquet_files[0]
-    logger.info(f"parquet_shard {dataset_name}: baixando {first_shard}...")
-    try:
-        local_path = hf_hub_download(
-            repo_id=dataset_name,
-            filename=first_shard,
-            repo_type="dataset",
-            token=hf_token,
-        )
-    except Exception as e:
-        logger.warning(f"parquet_shard {dataset_name}: download failed: {e}")
-        return
-
-    # Itera com pyarrow.iter_batches — método eficiente que lê batches pequenos
-    # do parquet sem materializar o row group inteiro na memória.
-    # BUG FIX V13.9.2-finetune-v2: read_row_group() materializa TODAS as ~1M
-    # rows de uma vez (lento + ~1GB RAM). iter_batches(batch_size=10) lê só
-    # 10 rows por vez (rápido + ~10KB RAM).
-    try:
-        pf = pq.ParquetFile(local_path)
-        count = 0
-        # batch_read_size: nº de rows por batch (pequeno = menos RAM)
-        for batch in pf.iter_batches(batch_size=100):
-            if count >= max_samples:
-                break
-            cols = batch.schema.names
-            # to_pylist() é eficiente para batches pequenos
-            pydict = batch.to_pydict()
-            n_rows = len(pydict[cols[0]]) if cols else 0
-            for i in range(n_rows):
-                if count >= max_samples:
-                    break
-                row = {col: pydict[col][i] for col in cols}
-                yield row
-                count += 1
-    except Exception as e:
-        logger.warning(f"parquet_shard {dataset_name}: pyarrow iter failed: {e}")
-        return
-
-    logger.info(f"parquet_shard {dataset_name}: streamou {count} amostras do shard {first_shard}")
-
-
-@dataclass
-class ProcessedSample:
-    """Uma amostra processada: texto bruto + metadados."""
-    raw_text: str
-    dataset_name: str
-    sample_idx: int
-    text_field: str = ""
-    label_text: str = ""
-
-
-def _extract_field(sample: Dict[str, Any], candidate_fields: List[str]) -> Optional[str]:
-    """Extrai texto da primeira field disponível no sample.
-
-    Suporta:
-    - strings diretas
-    - listas de strings (join com espaço — útil para datasets tokenizados
-      como adalbertojunior/punctuation-ptbr onde "tokens": ["hoje","a",...])
-    - listas de dicts (chat messages — concatena role: content)
-    - dicts aninhados (pega text/content/body)
-    """
-    for field_name in candidate_fields:
-        if field_name in sample:
-            val = sample[field_name]
-            if val is None:
-                continue
-            if isinstance(val, str):
-                if val.strip():
-                    return val
-            elif isinstance(val, list):
-                # V6.5-7ds: se for lista de strings, join com espaços
-                # (ex: adalbertojunior/punctuation-ptbr tokens field)
-                string_parts = []
-                chat_parts = []
-                all_strings = True
-                for item in val:
-                    if isinstance(item, str):
-                        if item.strip():
-                            string_parts.append(item)
-                    elif isinstance(item, (int, float)):
-                        # tokens podem vir como ints (token IDs) — skip
-                        all_strings = False
-                        break
-                    elif isinstance(item, dict):
-                        all_strings = False
-                        role = item.get("role", "")
-                        content = item.get("content", "")
-                        if isinstance(content, str) and content.strip():
-                            chat_parts.append(f"{role}: {content}")
-                    else:
-                        all_strings = False
-                if all_strings and string_parts:
-                    joined = " ".join(string_parts).strip()
-                    if joined:
-                        return joined
-                if chat_parts:
-                    return "\n".join(chat_parts)
-            elif isinstance(val, dict):
-                # Pode ser nested
-                text = val.get("text") or val.get("content") or val.get("body")
-                if isinstance(text, str) and text.strip():
-                    return text
-    return None
-
-
-def _normalize_sample(sample: Dict[str, Any], dataset_name: str, idx: int) -> Optional[ProcessedSample]:
-    """Normaliza uma amostra bruta em ProcessedSample."""
-    fmt = DATASET_FORMATS.get(dataset_name, {})
-    text_fields = fmt.get("text_fields", ["text"])
-    label_fields = fmt.get("label_fields", [])
-
-    text = _extract_field(sample, text_fields)
-    if not text or len(text.strip()) < 10:  # skip very short
-        return None
-
-    label = ""
-    if label_fields:
-        label = _extract_field(sample, label_fields) or ""
-
-    # Se tem label, aplica template apropriado
-    template = fmt.get("format_template")
-    full_text = text
-
-    # V13.9.2-finetune-v3: para o template instruction_response, exige ambos
-    # os lados (instruction + response). Samples sem response são descartados
-    # para evitar treinar em texto parcial.
-    if template == "instruction_response" and not label:
-        return None
-
-    if label:
-        # V13.9.2-finetune-v3: format_template unificado "### Instruction:/### Response:"
-        # quando o dataset declarar explicitamente. Caso contrário, mantém o
-        # comportamento legado (concatenação simples text\nlabel) para os 9
-        # datasets base — retrocompatibilidade total.
-        if template == "instruction_response":
-            instr_prefix = fmt.get("instruction_prefix", "")
-            if instr_prefix:
-                full_text = (
-                    f"### Instruction:\n{instr_prefix}\n{text}\n\n"
-                    f"### Response:\n{label}"
-                )
-            else:
-                full_text = (
-                    f"### Instruction:\n{text}\n\n"
-                    f"### Response:\n{label}"
-                )
-        elif template == "title_text_concat":
-            # V6.5-7ds: para datasets de notícias (iara-project, manoela),
-            # o campo "text" (corpo da notícia) é o conteúdo principal e
-            # "category" (categoria) é o contexto temático. Formatamos como
-            # "TEXTO: <corpo>\nCATEGORIA: <categoria>".
-            full_text = f"TEXTO: {text}\nCATEGORIA: {label}"
-        else:
-            # Comportamento legado (9 datasets base): concatenação simples
-            full_text = f"{text}\n{label}"
-
-    return ProcessedSample(
-        raw_text=full_text,
-        dataset_name=dataset_name,
-        sample_idx=idx,
-        text_field=text_fields[0] if text_fields else "",
-        label_text=label,
-    )
-
-
-def _load_dataset_streaming(
-    dataset_name: str,
-    split: str = "train",
-    config: Optional[str] = None,
-    hf_token: Optional[str] = None,
-):
-    """Carrega dataset em modo streaming.
-
-    BUG FIX V13.9.1: Removido trust_remote_code (deprecated).
-    Se o dataset não carregar com split especificado, tenta 'train' como fallback.
-    BUG FIX V13.9.2: Adicionado fallback para datasets com script (carolina-c4ai).
-    """
-    from datasets import load_dataset
-
-    # Lista de splits para tentar, em ordem
-    splits_to_try = [split]
-    if split != "train":
-        splits_to_try.append("train")
-    # Adiciona outros splits comuns
-    for s in ["test", "validation", "cot", "tir"]:
-        if s not in splits_to_try:
-            splits_to_try.append(s)
-
-    fmt = DATASET_FORMATS.get(dataset_name, {})
-    fallback_name = fmt.get("fallback")
-    datasets_to_try = [dataset_name]
-    if fallback_name:
-        datasets_to_try.append(fallback_name)
-
-    for try_ds_name in datasets_to_try:
-        for try_split in splits_to_try:
-            try:
-                if config:
-                    ds = load_dataset(
-                        try_ds_name, config, split=try_split, streaming=True,
-                        token=hf_token,
-                    )
-                else:
-                    ds = load_dataset(
-                        try_ds_name, split=try_split, streaming=True,
-                        token=hf_token,
-                    )
-                logger.info(f"Loaded {try_ds_name} split={try_split}")
-                return ds
-            except Exception as e:
-                err_str = str(e).lower()
-                # If "Bad split" error, try next split
-                if "bad split" in err_str or "available splits" in err_str:
-                    logger.info(f"  Split {try_split} not available for {try_ds_name}, trying next")
-                    continue
-                # If "scripts no longer supported", try next dataset (fallback)
-                if "scripts are no longer supported" in err_str and fallback_name:
-                    logger.info(f"  {try_ds_name} uses script — trying fallback {fallback_name}")
-                    break
-                # For other errors, log and try next split
-                logger.warning(f"Failed to load {try_ds_name} split={try_split}: {str(e)[:120]}")
-                continue
-
-    logger.warning(f"All split attempts failed for {dataset_name} (will be skipped)")
-    return None
-
-
-def stream_dataset(
-    dataset_name: str,
-    max_samples: int = 500,
-    hf_token: Optional[str] = None,
-) -> Iterator[ProcessedSample]:
-    """Faz streaming de um dataset, retornando até max_samples amostras.
-
-    Args:
-        dataset_name: nome do dataset no HuggingFace
-        max_samples: número máximo de amostras a retornar
-        hf_token: token HF opcional
-
-    Yields:
-        ProcessedSample
-    """
-    fmt = DATASET_FORMATS.get(dataset_name)
-    if fmt is None:
-        logger.error(f"Unknown dataset: {dataset_name}")
-        return
-
-    # ── V13.9.2-carolina: bypass do script Python ─────────────────────────
-    # O dataset carolina-c4ai/corpus-carolina usa um script Python como loader,
-    # que não é suportado por datasets 5.0+. Streamamos os XML.gz diretamente.
-    if dataset_name == "carolina-c4ai/corpus-carolina":
-        logger.info(f"Carolina: usando carregador direto (bypass do script Python)")
-        count = 0
-        try:
-            for raw_sample in _stream_carolina_direct(
-                max_samples=max_samples, hf_token=hf_token
-            ):
-                processed = _normalize_sample(raw_sample, dataset_name, count)
-                if processed is not None:
-                    yield processed
-                    count += 1
-        except Exception as e:
-            logger.warning(f"Carolina direct stream error at sample {count}: {e}")
-            return
-        logger.info(f"Streamed {count} samples from {dataset_name} (direct XML)")
-        return
-
-    # ── V13.9.2-finetune-v2: loaders alternativos para parquets grandes ───
-    # Alguns datasets (cnmoro, strak2005/bratao) têm parquets tão grandes que
-    # datasets.load_dataset(streaming=True) demora minutos para materializar
-    # a primeira amostra. Usamos loaders alternativos especificados no campo
-    # "loader" do formato.
-    loader = fmt.get("loader")
-    if loader == "rows_api":
-        logger.info(f"{dataset_name}: usando rows_api loader (datasets-server HTTP)")
-        count = 0
-        try:
-            for raw_sample in _stream_via_rows_api(
-                dataset_name, max_samples=max_samples, hf_token=hf_token,
-                config=fmt.get("config"),
-                split=fmt.get("split", "train"),
-            ):
-                processed = _normalize_sample(raw_sample, dataset_name, count)
-                if processed is not None:
-                    yield processed
-                    count += 1
-        except Exception as e:
-            logger.warning(f"rows_api stream error at sample {count}: {e}")
-            return
-        logger.info(f"Streamed {count} samples from {dataset_name} (rows_api)")
-        return
-
-    if loader == "parquet_first_shard":
-        logger.info(f"{dataset_name}: usando parquet_first_shard loader")
-        count = 0
-        try:
-            for raw_sample in _stream_via_parquet_first_shard(
-                dataset_name, max_samples=max_samples, hf_token=hf_token
-            ):
-                processed = _normalize_sample(raw_sample, dataset_name, count)
-                if processed is not None:
-                    yield processed
-                    count += 1
-        except Exception as e:
-            logger.warning(f"parquet_shard stream error at sample {count}: {e}")
-            # Tenta fallback (ex: strak2005 -> bratao)
-            fallback_name = fmt.get("fallback")
-            if fallback_name and fallback_name != dataset_name:
-                logger.info(f"Tentando fallback: {fallback_name}")
-                fallback_fmt = DATASET_FORMATS.get(fallback_name, {})
-                # Recursão respeitando o loader do fallback
-                yield from stream_dataset(
-                    fallback_name, max_samples=max_samples, hf_token=hf_token
-                )
-            return
-        logger.info(f"Streamed {count} samples from {dataset_name} (parquet_first_shard)")
-        return
-
-    ds = _load_dataset_streaming(
-        dataset_name,
-        split=fmt.get("split", "train"),
-        config=fmt.get("config"),
-        hf_token=hf_token,
-    )
-    if ds is None:
-        return
-
-    count = 0
-    try:
-        for idx, sample in enumerate(ds):
-            if count >= max_samples:
-                break
-            processed = _normalize_sample(sample, dataset_name, idx)
-            if processed is not None:
-                yield processed
-                count += 1
-    except Exception as e:
-        logger.warning(f"Error streaming {dataset_name} at sample {idx}: {e}")
-        return
-
-    logger.info(f"Streamed {count} samples from {dataset_name}")
-
-
-def stream_all_datasets(
-    datasets: Optional[List[str]] = None,
-    max_samples_per_dataset: int = 500,
-    hf_token: Optional[str] = None,
-) -> Iterator[ProcessedSample]:
-    """Faz streaming sequencial de múltiplos datasets.
-
-    Args:
-        datasets: lista de nomes (default: DEFAULT_DATASETS)
-        max_samples_per_dataset: limite por dataset
-        hf_token: token HF opcional
-
-    Yields:
-        ProcessedSample
-    """
-    if datasets is None:
-        datasets = DEFAULT_DATASETS
-
-    for ds_name in datasets:
-        logger.info(f"Starting dataset: {ds_name}")
-        yield from stream_dataset(
-            ds_name,
-            max_samples=max_samples_per_dataset,
-            hf_token=hf_token,
-        )
-
-
-def collate_samples_to_tensors(
-    samples: List[ProcessedSample],
-    tokenizer,
-    max_seq_len: int = 64,
-    pad_token_id: int = 0,
-) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
-    """Converte lista de ProcessedSample em tensors para treino.
-
-    BUG FIX V13.9.1: Tokenizer.encode() retorna 'Encoding' object, não list.
-    Use .ids attribute to get token IDs.
-
-    Returns:
-        input_ids: (B, T)
-        attention_mask: (B, T)
-        labels: (B, T) — same as input_ids (LM training)
-    """
-    batch_input_ids = []
-    batch_attn_mask = []
-
-    for sample in samples:
-        # Tokenize
-        encoding = tokenizer.encode(sample.raw_text, add_special_tokens=True)
-
-        # BUG FIX: Handle both `tokenizers.Encoding` and `transformers` tokenizer outputs
-        if hasattr(encoding, 'ids'):
-            tokens = encoding.ids  # tokenizers library
-        elif isinstance(encoding, list):
-            tokens = encoding  # already a list of ints
-        elif hasattr(encoding, 'input_ids'):
-            tokens = encoding.input_ids  # transformers BatchEncoding
-        else:
-            tokens = list(encoding)  # fallback
-
-        # Truncate to max_seq_len
-        if len(tokens) > max_seq_len:
-            tokens = tokens[:max_seq_len]
-        # Pad
-        attn_mask = [1] * len(tokens) + [0] * (max_seq_len - len(tokens))
-        tokens = list(tokens) + [pad_token_id] * (max_seq_len - len(tokens))
-        batch_input_ids.append(tokens)
-        batch_attn_mask.append(attn_mask)
-
-    input_ids = torch.tensor(batch_input_ids, dtype=torch.long)
-    attention_mask = torch.tensor(batch_attn_mask, dtype=torch.long)
-    labels = input_ids.clone()
-
-    return input_ids, attention_mask, labels
-
-
-# ═══════════════════════════════════════════════════════════════════════════
-# Self-test (NÃO enviar para HuggingFace)
-# ═══════════════════════════════════════════════════════════════════════════
-if __name__ == "__main__":
-    print("=== Streaming Datasets V13.9.1 Self-Test ===\n")
-
-    # Test 1: Test field extraction
-    print("Test 1: Field extraction")
-    sample_chat = {
-        "messages": [
-            {"role": "user", "content": "Olá"},
-            {"role": "assistant", "content": "Como posso ajudar?"},
-        ]
-    }
-    text = _extract_field(sample_chat, ["messages", "text"])
-    print(f"  Chat text: {text!r}")
-    assert text and "Olá" in text, "Field extraction failed for chat"
-
-    sample_text = {"text": "Hello world"}
-    text = _extract_field(sample_text, ["text", "content"])
-    assert text == "Hello world"
-    print(f"  Text field: {text!r}")
-    print(f"  OK\n")
-
-    # Test 2: Test list of datasets
-    print("Test 2: Dataset list")
-    print(f"  Configured datasets ({len(DEFAULT_DATASETS)}):")
-    for i, ds in enumerate(DEFAULT_DATASETS, 1):
-        print(f"    {i}. {ds}")
-    assert len(DEFAULT_DATASETS) == 14
-    # V6.5-V2: verifica sequência CONHECIMENTO (8 datasets) + PUNICAO (1)
-    assert len(CONHECIMENTO_DATASETS_V2) == 8
-    assert PUNICAO_DATASET_V2 == "BrunoN-Dev/corpus-ptbr-v1"
-    print(f"  V2 CONHECIMENTO datasets: {len(CONHECIMENTO_DATASETS_V2)}")
-    print(f"  V2 PUNICAO dataset: {PUNICAO_DATASET_V2}")
-    print(f"  OK\n")
-
-    print("=== ALL STREAMING TESTS PASSED ===")

v6_5_v2_attention_eval.json

@@ -1,21 +0,0 @@
-{
-  "evaluation": "attention_v65_v2",
-  "user_requirement": "verificar se o mecanismo de atenção está ativo e acessado logicamente funcional",
-  "metrics": {
-    "active": true,
-    "n_calls": 500,
-    "n_errors": 0,
-    "last_norm_in": 108.67192077636719,
-    "last_norm_out": 112.93913269042969,
-    "last_attn_activated": true,
-    "last_attn_diff_norm": 114.95559692382812,
-    "n_heads": 8,
-    "logic_functional": true
-  },
-  "active": true,
-  "logic_functional": true,
-  "n_calls": 500,
-  "n_errors": 0,
-  "n_heads": 8,
-  "assessment": "PASS"
-}

v6_5_v2_phases_eval.json

@@ -1,1414 +0,0 @@
-{
-  "fase_1_conhecimento": {
-    "phase": "CONHECIMENTO",
-    "total_samples": 500,
-    "meta_minima": 8000,
-    "meta_atingida": false,
-    "samples_per_dataset": {
-      "dominguesm/restore-punctuation-ptbr-dataset": 500,
-      "carolina-c4ai/corpus-carolina": 0,
-      "CEIA-POSITIVO/ultrachat_br_clustred_balanced_v1": 0,
-      "dominguesm/Canarim-Instruct-PTBR-Dataset": 0,
-      "adalbertojunior/punctuation-ptbr": 0,
-      "iara-project/news-articles-ptbr-dataset": 0,
-      "manoela/noticias_ptbr": 0,
-      "BrunoN-Dev/corpus-ptbr-v1": 0
-    },
-    "streaming_failures": {
-      "dominguesm/restore-punctuation-ptbr-dataset": 0,
-      "carolina-c4ai/corpus-carolina": 0,
-      "CEIA-POSITIVO/ultrachat_br_clustred_balanced_v1": 0,
-      "dominguesm/Canarim-Instruct-PTBR-Dataset": 0,
-      "adalbertojunior/punctuation-ptbr": 0,
-      "iara-project/news-articles-ptbr-dataset": 0,
-      "manoela/noticias_ptbr": 0,
-      "BrunoN-Dev/corpus-ptbr-v1": 0
-    },
-    "storage_critical_stopped": true,
-    "elapsed_s": 37.09058332443237,
-    "n_steps": 35,
-    "n_chunks": 5,
-    "accuracies_log": [
-      {
-        "step": 0,
-        "dataset": "dominguesm/restore-punctuation-ptbr-dataset",
-        "chunk_idx": 1,
-        "batch_size": 16,
-        "accuracy": 0.9375,
-        "phase": "CONHECIMENTO",
-        "buffer_size": 16,
-        "training_ready": true,
-        "elapsed_ms": 383.3961486816406
-      },
-      {
-        "step": 1,
-        "dataset": "dominguesm/restore-punctuation-ptbr-dataset",
-        "chunk_idx": 1,
-        "batch_size": 16,
-        "accuracy": 0.9375,
-        "phase": "CONHECIMENTO",
-        "buffer_size": 32,
-        "training_ready": true,
-        "elapsed_ms": 105.80253601074219
-      },
-      {
-        "step": 2,
-        "dataset": "dominguesm/restore-punctuation-ptbr-dataset",
-        "chunk_idx": 1,
-        "batch_size": 16,
-        "accuracy": 0.9791666666666666,
-        "phase": "CONHECIMENTO",
-        "buffer_size": 48,
-        "training_ready": true,
-        "elapsed_ms": 146.81077003479004
-      },
-      {
-        "step": 3,
-        "dataset": "dominguesm/restore-punctuation-ptbr-dataset",
-        "chunk_idx": 1,
-        "batch_size": 16,
-        "accuracy": 0.984375,
-        "phase": "CONHECIMENTO",
-        "buffer_size": 64,
-        "training_ready": true,
-        "elapsed_ms": 166.37229919433594
-      },
-      {
-        "step": 4,
-        "dataset": "dominguesm/restore-punctuation-ptbr-dataset",
-        "chunk_idx": 1,
-        "batch_size": 16,
-        "accuracy": 1.0,
-        "phase": "CONHECIMENTO",
-        "buffer_size": 80,
-        "training_ready": true,
-        "elapsed_ms": 226.5923023223877
-      },
-      {
-        "step": 5,
-        "dataset": "dominguesm/restore-punctuation-ptbr-dataset",
-        "chunk_idx": 1,
-        "batch_size": 16,
-        "accuracy": 0.9895833333333334,
-        "phase": "CONHECIMENTO",
-        "buffer_size": 96,
-        "training_ready": true,
-        "elapsed_ms": 256.2379837036133
-      },
-      {
-        "step": 6,
-        "dataset": "dominguesm/restore-punctuation-ptbr-dataset",
-        "chunk_idx": 1,
-        "batch_size": 4,
-        "accuracy": 0.98,
-        "phase": "CONHECIMENTO",
-        "buffer_size": 100,
-        "training_ready": true,
-        "elapsed_ms": 236.21749877929688
-      },
-      {
-        "step": 7,
-        "dataset": "dominguesm/restore-punctuation-ptbr-dataset",
-        "chunk_idx": 2,
-        "batch_size": 16,
-        "accuracy": 0.9482758620689655,
-        "phase": "CONHECIMENTO",
-        "buffer_size": 116,
-        "training_ready": true,
-        "elapsed_ms": 329.33592796325684
-      },
-      {
-        "step": 8,
-        "dataset": "dominguesm/restore-punctuation-ptbr-dataset",
-        "chunk_idx": 2,
-        "batch_size": 16,
-        "accuracy": 0.9393939393939394,
-        "phase": "CONHECIMENTO",
-        "buffer_size": 132,
-        "training_ready": true,
-        "elapsed_ms": 356.5490245819092
-      },
-      {
-        "step": 9,
-        "dataset": "dominguesm/restore-punctuation-ptbr-dataset",
-        "chunk_idx": 2,
-        "batch_size": 16,
-        "accuracy": 0.9324324324324325,
-        "phase": "CONHECIMENTO",
-        "buffer_size": 148,
-        "training_ready": true,
-        "elapsed_ms": 368.1018352508545
-      },
-      {
-        "step": 10,
-        "dataset": "dominguesm/restore-punctuation-ptbr-dataset",
-        "chunk_idx": 2,
-        "batch_size": 16,
-        "accuracy": 0.9390243902439024,
-        "phase": "CONHECIMENTO",
-        "buffer_size": 164,
-        "training_ready": true,
-        "elapsed_ms": 498.46649169921875
-      },
-      {
-        "step": 11,
-        "dataset": "dominguesm/restore-punctuation-ptbr-dataset",
-        "chunk_idx": 2,
-        "batch_size": 16,
-        "accuracy": 0.9388888888888889,
-        "phase": "CONHECIMENTO",
-        "buffer_size": 180,
-        "training_ready": true,
-        "elapsed_ms": 428.97915840148926
-      },
-      {
-        "step": 12,
-        "dataset": "dominguesm/restore-punctuation-ptbr-dataset",
-        "chunk_idx": 2,
-        "batch_size": 16,
-        "accuracy": 0.9336734693877551,
-        "phase": "CONHECIMENTO",
-        "buffer_size": 196,
-        "training_ready": true,
-        "elapsed_ms": 450.5326747894287
-      },
-      {
-        "step": 13,
-        "dataset": "dominguesm/restore-punctuation-ptbr-dataset",
-        "chunk_idx": 2,
-        "batch_size": 4,
-        "accuracy": 0.935,
-        "phase": "CONHECIMENTO",
-        "buffer_size": 200,
-        "training_ready": true,
-        "elapsed_ms": 688.495397567749
-      },
-      {
-        "step": 14,
-        "dataset": "dominguesm/restore-punctuation-ptbr-dataset",
-        "chunk_idx": 3,
-        "batch_size": 16,
-        "accuracy": 0.8958333333333334,
-        "phase": "CONHECIMENTO",
-        "buffer_size": 144,
-        "training_ready": true,
-        "elapsed_ms": 360.4564666748047
-      },
-      {
-        "step": 15,
-        "dataset": "dominguesm/restore-punctuation-ptbr-dataset",
-        "chunk_idx": 3,
-        "batch_size": 16,
-        "accuracy": 0.9,
-        "phase": "CONHECIMENTO",
-        "buffer_size": 160,
-        "training_ready": true,
-        "elapsed_ms": 377.14409828186035
-      },
-      {
-        "step": 16,
-        "dataset": "dominguesm/restore-punctuation-ptbr-dataset",
-        "chunk_idx": 3,
-        "batch_size": 16,
-        "accuracy": 0.8977272727272727,
-        "phase": "CONHECIMENTO",
-        "buffer_size": 176,
-        "training_ready": true,
-        "elapsed_ms": 533.5609912872314
-      },
-      {
-        "step": 17,
-        "dataset": "dominguesm/restore-punctuation-ptbr-dataset",
-        "chunk_idx": 3,
-        "batch_size": 16,
-        "accuracy": 0.8958333333333334,
-        "phase": "CONHECIMENTO",
-        "buffer_size": 192,
-        "training_ready": true,
-        "elapsed_ms": 428.7114143371582
-      },
-      {
-        "step": 18,
-        "dataset": "dominguesm/restore-punctuation-ptbr-dataset",
-        "chunk_idx": 3,
-        "batch_size": 16,
-        "accuracy": 0.8990384615384616,
-        "phase": "CONHECIMENTO",
-        "buffer_size": 208,
-        "training_ready": true,
-        "elapsed_ms": 477.15067863464355
-      },
-      {
-        "step": 19,
-        "dataset": "dominguesm/restore-punctuation-ptbr-dataset",
-        "chunk_idx": 3,
-        "batch_size": 16,
-        "accuracy": 0.8928571428571429,
-        "phase": "CONHECIMENTO",
-        "buffer_size": 224,
-        "training_ready": true,
-        "elapsed_ms": 530.5352210998535
-      },
-      {
-        "step": 20,
-        "dataset": "dominguesm/restore-punctuation-ptbr-dataset",
-        "chunk_idx": 3,
-        "batch_size": 4,
-        "accuracy": 0.8903508771929824,
-        "phase": "CONHECIMENTO",
-        "buffer_size": 228,
-        "training_ready": true,
-        "elapsed_ms": 529.3123722076416
-      },
-      {
-        "step": 21,
-        "dataset": "dominguesm/restore-punctuation-ptbr-dataset",
-        "chunk_idx": 4,
-        "batch_size": 16,
-        "accuracy": 0.8958333333333334,
-        "phase": "CONHECIMENTO",
-        "buffer_size": 144,
-        "training_ready": true,
-        "elapsed_ms": 378.77631187438965
-      },
-      {
-        "step": 22,
-        "dataset": "dominguesm/restore-punctuation-ptbr-dataset",
-        "chunk_idx": 4,
-        "batch_size": 16,
-        "accuracy": 0.9,
-        "phase": "CONHECIMENTO",
-        "buffer_size": 160,
-        "training_ready": true,
-        "elapsed_ms": 772.2029685974121
-      },
-      {
-        "step": 23,
-        "dataset": "dominguesm/restore-punctuation-ptbr-dataset",
-        "chunk_idx": 4,
-        "batch_size": 16,
-        "accuracy": 0.9034090909090909,
-        "phase": "CONHECIMENTO",
-        "buffer_size": 176,
-        "training_ready": true,
-        "elapsed_ms": 374.96161460876465
-      },
-      {
-        "step": 24,
-        "dataset": "dominguesm/restore-punctuation-ptbr-dataset",
-        "chunk_idx": 4,
-        "batch_size": 16,
-        "accuracy": 0.9114583333333334,
-        "phase": "CONHECIMENTO",
-        "buffer_size": 192,
-        "training_ready": true,
-        "elapsed_ms": 416.3072109222412
-      },
-      {
-        "step": 25,
-        "dataset": "dominguesm/restore-punctuation-ptbr-dataset",
-        "chunk_idx": 4,
-        "batch_size": 16,
-        "accuracy": 0.9086538461538461,
-        "phase": "CONHECIMENTO",
-        "buffer_size": 208,
-        "training_ready": true,
-        "elapsed_ms": 850.2218723297119
-      },
-      {
-        "step": 26,
-        "dataset": "dominguesm/restore-punctuation-ptbr-dataset",
-        "chunk_idx": 4,
-        "batch_size": 16,
-        "accuracy": 0.9151785714285714,
-        "phase": "CONHECIMENTO",
-        "buffer_size": 224,
-        "training_ready": true,
-        "elapsed_ms": 491.5575981140137
-      },
-      {
-        "step": 27,
-        "dataset": "dominguesm/restore-punctuation-ptbr-dataset",
-        "chunk_idx": 4,
-        "batch_size": 4,
-        "accuracy": 0.9122807017543859,
-        "phase": "CONHECIMENTO",
-        "buffer_size": 228,
-        "training_ready": true,
-        "elapsed_ms": 455.5957317352295
-      },
-      {
-        "step": 28,
-        "dataset": "dominguesm/restore-punctuation-ptbr-dataset",
-        "chunk_idx": 5,
-        "batch_size": 16,
-        "accuracy": 0.9097222222222222,
-        "phase": "CONHECIMENTO",
-        "buffer_size": 144,
-        "training_ready": true,
-        "elapsed_ms": 335.35003662109375
-      },
-      {
-        "step": 29,
-        "dataset": "dominguesm/restore-punctuation-ptbr-dataset",
-        "chunk_idx": 5,
-        "batch_size": 16,
-        "accuracy": 0.9125,
-        "phase": "CONHECIMENTO",
-        "buffer_size": 160,
-        "training_ready": true,
-        "elapsed_ms": 337.5859260559082
-      },
-      {
-        "step": 30,
-        "dataset": "dominguesm/restore-punctuation-ptbr-dataset",
-        "chunk_idx": 5,
-        "batch_size": 16,
-        "accuracy": 0.8977272727272727,
-        "phase": "CONHECIMENTO",
-        "buffer_size": 176,
-        "training_ready": true,
-        "elapsed_ms": 411.2229347229004
-      },
-      {
-        "step": 31,
-        "dataset": "dominguesm/restore-punctuation-ptbr-dataset",
-        "chunk_idx": 5,
-        "batch_size": 16,
-        "accuracy": 0.90625,
-        "phase": "CONHECIMENTO",
-        "buffer_size": 192,
-        "training_ready": true,
-        "elapsed_ms": 742.3462867736816
-      },
-      {
-        "step": 32,
-        "dataset": "dominguesm/restore-punctuation-ptbr-dataset",
-        "chunk_idx": 5,
-        "batch_size": 16,
-        "accuracy": 0.9086538461538461,
-        "phase": "CONHECIMENTO",
-        "buffer_size": 208,
-        "training_ready": true,
-        "elapsed_ms": 1044.4471836090088
-      },
-      {
-        "step": 33,
-        "dataset": "dominguesm/restore-punctuation-ptbr-dataset",
-        "chunk_idx": 5,
-        "batch_size": 16,
-        "accuracy": 0.90625,
-        "phase": "CONHECIMENTO",
-        "buffer_size": 224,
-        "training_ready": true,
-        "elapsed_ms": 510.6990337371826
-      },
-      {
-        "step": 34,
-        "dataset": "dominguesm/restore-punctuation-ptbr-dataset",
-        "chunk_idx": 5,
-        "batch_size": 4,
-        "accuracy": 0.8991228070175439,
-        "phase": "CONHECIMENTO",
-        "buffer_size": 228,
-        "training_ready": true,
-        "elapsed_ms": 536.2398624420166
-      }
-    ],
-    "attention_log": [
-      {
-        "step": 12,
-        "dataset": "dominguesm/restore-punctuation-ptbr-dataset",
-        "n_calls": 196,
-        "n_errors": 0,
-        "last_attn_activated": true,
-        "logic_functional": true
-      },
-      {
-        "step": 14,
-        "dataset": "dominguesm/restore-punctuation-ptbr-dataset",
-        "n_calls": 216,
-        "n_errors": 0,
-        "last_attn_activated": true,
-        "logic_functional": true
-      },
-      {
-        "step": 15,
-        "dataset": "dominguesm/restore-punctuation-ptbr-dataset",
-        "n_calls": 232,
-        "n_errors": 0,
-        "last_attn_activated": true,
-        "logic_functional": true
-      },
-      {
-        "step": 16,
-        "dataset": "dominguesm/restore-punctuation-ptbr-dataset",
-        "n_calls": 248,
-        "n_errors": 0,
-        "last_attn_activated": true,
-        "logic_functional": true
-      },
-      {
-        "step": 17,
-        "dataset": "dominguesm/restore-punctuation-ptbr-dataset",
-        "n_calls": 264,
-        "n_errors": 0,
-        "last_attn_activated": true,
-        "logic_functional": true
-      },
-      {
-        "step": 18,
-        "dataset": "dominguesm/restore-punctuation-ptbr-dataset",
-        "n_calls": 280,
-        "n_errors": 0,
-        "last_attn_activated": true,
-        "logic_functional": true
-      },
-      {
-        "step": 19,
-        "dataset": "dominguesm/restore-punctuation-ptbr-dataset",
-        "n_calls": 296,
-        "n_errors": 0,
-        "last_attn_activated": true,
-        "logic_functional": true
-      },
-      {
-        "step": 21,
-        "dataset": "dominguesm/restore-punctuation-ptbr-dataset",
-        "n_calls": 316,
-        "n_errors": 0,
-        "last_attn_activated": true,
-        "logic_functional": true
-      },
-      {
-        "step": 22,
-        "dataset": "dominguesm/restore-punctuation-ptbr-dataset",
-        "n_calls": 332,
-        "n_errors": 0,
-        "last_attn_activated": true,
-        "logic_functional": true
-      },
-      {
-        "step": 23,
-        "dataset": "dominguesm/restore-punctuation-ptbr-dataset",
-        "n_calls": 348,
-        "n_errors": 0,
-        "last_attn_activated": true,
-        "logic_functional": true
-      },
-      {
-        "step": 24,
-        "dataset": "dominguesm/restore-punctuation-ptbr-dataset",
-        "n_calls": 364,
-        "n_errors": 0,
-        "last_attn_activated": true,
-        "logic_functional": true
-      },
-      {
-        "step": 25,
-        "dataset": "dominguesm/restore-punctuation-ptbr-dataset",
-        "n_calls": 380,
-        "n_errors": 0,
-        "last_attn_activated": true,
-        "logic_functional": true
-      },
-      {
-        "step": 26,
-        "dataset": "dominguesm/restore-punctuation-ptbr-dataset",
-        "n_calls": 396,
-        "n_errors": 0,
-        "last_attn_activated": true,
-        "logic_functional": true
-      },
-      {
-        "step": 28,
-        "dataset": "dominguesm/restore-punctuation-ptbr-dataset",
-        "n_calls": 416,
-        "n_errors": 0,
-        "last_attn_activated": true,
-        "logic_functional": true
-      },
-      {
-        "step": 29,
-        "dataset": "dominguesm/restore-punctuation-ptbr-dataset",
-        "n_calls": 432,
-        "n_errors": 0,
-        "last_attn_activated": true,
-        "logic_functional": true
-      },
-      {
-        "step": 30,
-        "dataset": "dominguesm/restore-punctuation-ptbr-dataset",
-        "n_calls": 448,
-        "n_errors": 0,
-        "last_attn_activated": true,
-        "logic_functional": true
-      },
-      {
-        "step": 31,
-        "dataset": "dominguesm/restore-punctuation-ptbr-dataset",
-        "n_calls": 464,
-        "n_errors": 0,
-        "last_attn_activated": true,
-        "logic_functional": true
-      },
-      {
-        "step": 32,
-        "dataset": "dominguesm/restore-punctuation-ptbr-dataset",
-        "n_calls": 480,
-        "n_errors": 0,
-        "last_attn_activated": true,
-        "logic_functional": true
-      },
-      {
-        "step": 33,
-        "dataset": "dominguesm/restore-punctuation-ptbr-dataset",
-        "n_calls": 496,
-        "n_errors": 0,
-        "last_attn_activated": true,
-        "logic_functional": true
-      },
-      {
-        "step": 34,
-        "dataset": "dominguesm/restore-punctuation-ptbr-dataset",
-        "n_calls": 500,
-        "n_errors": 0,
-        "last_attn_activated": true,
-        "logic_functional": true
-      }
-    ],
-    "som_metrics_log": [
-      {
-        "step": 7,
-        "chunk_idx": 1,
-        "dataset": "dominguesm/restore-punctuation-ptbr-dataset",
-        "quantization_error": 2.26032170758117e-05,
-        "topological_error": 0.880000002682209,
-        "kaski_lagus_error": 0.4445206419101888,
-        "explained_variance_share": 0.999451068570896,
-        "overall_health": "critical",
-        "n_failure_indicators": 3,
-        "failure_indicators": [
-          "topological_collapse_line",
-          "dead_neuron_rate_high_0.89",
-          "high_topological_error_0.88"
-        ],
-        "collapse_severity": "line",
-        "dead_neuron_rate": 0.8900462962962963,
-        "nan_detected": false,
-        "nan_reason": null,
-        "qe_was_nan": false,
-        "kl_was_nan": false,
-        "n_hypotheses_active": 0,
-        "n_hypotheses_total": 16,
-        "hypothesis_activation_rate": 0.0,
-        "mean_delta_norm": 1.891924029278399e-12,
-        "max_delta_norm": 2.1833108238000776e-12,
-        "n_active_neurons_bmu": 95,
-        "n_dead_neurons_bmu": 769,
-        "neuron_activation_rate": 0.1099537037037037,
-        "current_hyp_train_steps": 30,
-        "total_hyp_steps_executed": 0,
-        "n_train_hyp_calls": 0
-      },
-      {
-        "step": 14,
-        "chunk_idx": 2,
-        "dataset": "dominguesm/restore-punctuation-ptbr-dataset",
-        "quantization_error": 0.000569911440834403,
-        "topological_error": 0.9140625,
-        "kaski_lagus_error": 0.5460798889633685,
-        "explained_variance_share": 0.928539884025705,
-        "overall_health": "critical",
-        "n_failure_indicators": 3,
-        "failure_indicators": [
-          "topological_collapse_line",
-          "dead_neuron_rate_high_0.95",
-          "high_topological_error_0.91"
-        ],
-        "collapse_severity": "line",
-        "dead_neuron_rate": 0.9456018518518519,
-        "nan_detected": false,
-        "nan_reason": null,
-        "qe_was_nan": false,
-        "kl_was_nan": false,
-        "n_hypotheses_active": 0,
-        "n_hypotheses_total": 16,
-        "hypothesis_activation_rate": 0.0,
-        "mean_delta_norm": 1.570570957358866e-11,
-        "max_delta_norm": 1.831555566933485e-11,
-        "n_active_neurons_bmu": 47,
-        "n_dead_neurons_bmu": 817,
-        "neuron_activation_rate": 0.05439814814814815,
-        "current_hyp_train_steps": 30,
-        "total_hyp_steps_executed": 0,
-        "n_train_hyp_calls": 0
-      },
-      {
-        "step": 21,
-        "chunk_idx": 3,
-        "dataset": "dominguesm/restore-punctuation-ptbr-dataset",
-        "quantization_error": 0.005070194136351347,
-        "topological_error": 0.984375,
-        "kaski_lagus_error": 1.2844052385351477,
-        "explained_variance_share": 0.05886936612972116,
-        "overall_health": "critical",
-        "n_failure_indicators": 4,
-        "failure_indicators": [
-          "topological_collapse_line",
-          "dead_neuron_rate_high_0.99",
-          "low_explained_variance_0.06",
-          "high_topological_error_0.98"
-        ],
-        "collapse_severity": "line",
-        "dead_neuron_rate": 0.9884259259259259,
-        "nan_detected": false,
-        "nan_reason": null,
-        "qe_was_nan": false,
-        "kl_was_nan": false,
-        "n_hypotheses_active": 0,
-        "n_hypotheses_total": 16,
-        "hypothesis_activation_rate": 0.0,
-        "mean_delta_norm": 5.5481111892186075e-11,
-        "max_delta_norm": 6.390629692099026e-11,
-        "n_active_neurons_bmu": 10,
-        "n_dead_neurons_bmu": 854,
-        "neuron_activation_rate": 0.011574074074074073,
-        "current_hyp_train_steps": 30,
-        "total_hyp_steps_executed": 0,
-        "n_train_hyp_calls": 0
-      },
-      {
-        "step": 28,
-        "chunk_idx": 4,
-        "dataset": "dominguesm/restore-punctuation-ptbr-dataset",
-        "quantization_error": 0.012031015940010548,
-        "topological_error": 1.0,
-        "kaski_lagus_error": 2.3798458777851814,
-        "explained_variance_share": 0.0,
-        "overall_health": "critical",
-        "n_failure_indicators": 3,
-        "failure_indicators": [
-          "dead_neuron_rate_high_1.00",
-          "low_explained_variance_0.00",
-          "high_topological_error_1.00"
-        ],
-        "collapse_severity": "none",
-        "dead_neuron_rate": 0.9976851851851852,
-        "nan_detected": false,
-        "nan_reason": null,
-        "qe_was_nan": false,
-        "kl_was_nan": false,
-        "n_hypotheses_active": 0,
-        "n_hypotheses_total": 16,
-        "hypothesis_activation_rate": 0.0,
-        "mean_delta_norm": 6.429341642411046e-10,
-        "max_delta_norm": 6.918212669404511e-10,
-        "n_active_neurons_bmu": 2,
-        "n_dead_neurons_bmu": 862,
-        "neuron_activation_rate": 0.0023148148148148147,
-        "current_hyp_train_steps": 30,
-        "total_hyp_steps_executed": 0,
-        "n_train_hyp_calls": 0
-      },
-      {
-        "step": 35,
-        "chunk_idx": 5,
-        "dataset": "dominguesm/restore-punctuation-ptbr-dataset",
-        "quantization_error": 0.009746708907186985,
-        "topological_error": 1.0,
-        "kaski_lagus_error": 2.0229230836057637,
-        "explained_variance_share": 0.0,
-        "overall_health": "critical",
-        "n_failure_indicators": 3,
-        "failure_indicators": [
-          "dead_neuron_rate_high_1.00",
-          "low_explained_variance_0.00",
-          "high_topological_error_1.00"
-        ],
-        "collapse_severity": "none",
-        "dead_neuron_rate": 0.9976851851851852,
-        "nan_detected": false,
-        "nan_reason": null,
-        "qe_was_nan": false,
-        "kl_was_nan": false,
-        "n_hypotheses_active": 0,
-        "n_hypotheses_total": 16,
-        "hypothesis_activation_rate": 0.0,
-        "mean_delta_norm": 6.375817547532581e-10,
-        "max_delta_norm": 7.304411520081544e-10,
-        "n_active_neurons_bmu": 2,
-        "n_dead_neurons_bmu": 862,
-        "neuron_activation_rate": 0.0023148148148148147,
-        "current_hyp_train_steps": 30,
-        "total_hyp_steps_executed": 0,
-        "n_train_hyp_calls": 0
-      }
-    ],
-    "som_metrics_final": {
-      "quantization_error": 0.009746708907186985,
-      "topological_error": 1.0,
-      "kaski_lagus_error": 2.0229230836057637,
-      "explained_variance_share": 0.0,
-      "topological_collapse": {
-        "collapsed_to_point": false,
-        "collapsed_to_line": false,
-        "effective_rank": 3.9666486969891515,
-        "singular_values": [
-          0.7090448141098022,
-          0.6143536567687988,
-          0.5655514597892761,
-          0.4965478777885437
-        ],
-        "severity": "none"
-      },
-      "dead_neuron_rate": {
-        "dead_neuron_rate": 0.9976851851851852,
-        "n_dead": 862,
-        "n_active": 2,
-        "n_total": 864,
-        "dead_indices": [
-          [
-            0,
-            0,
-            0,
-            0
-          ],
-          [
-            0,
-            0,
-            0,
-            1
-          ],
-          [
-            0,
-            0,
-            0,
-            2
-          ],
-          [
-            0,
-            0,
-            0,
-            3
-          ],
-          [
-            0,
-            0,
-            1,
-            0
-          ],
-          [
-            0,
-            0,
-            1,
-            1
-          ],
-          [
-            0,
-            0,
-            1,
-            2
-          ],
-          [
-            0,
-            0,
-            1,
-            3
-          ],
-          [
-            0,
-            0,
-            2,
-            0
-          ],
-          [
-            0,
-            0,
-            2,
-            1
-          ],
-          [
-            0,
-            0,
-            2,
-            2
-          ],
-          [
-            0,
-            0,
-            2,
-            3
-          ],
-          [
-            0,
-            0,
-            3,
-            0
-          ],
-          [
-            0,
-            0,
-            3,
-            1
-          ],
-          [
-            0,
-            0,
-            3,
-            2
-          ],
-          [
-            0,
-            0,
-            3,
-            3
-          ],
-          [
-            0,
-            0,
-            4,
-            0
-          ],
-          [
-            0,
-            0,
-            4,
-            1
-          ],
-          [
-            0,
-            0,
-            4,
-            2
-          ],
-          [
-            0,
-            0,
-            4,
-            3
-          ],
-          [
-            0,
-            0,
-            5,
-            0
-          ],
-          [
-            0,
-            0,
-            5,
-            1
-          ],
-          [
-            0,
-            0,
-            5,
-            2
-          ],
-          [
-            0,
-            0,
-            5,
-            3
-          ],
-          [
-            0,
-            1,
-            0,
-            0
-          ],
-          [
-            0,
-            1,
-            0,
-            1
-          ],
-          [
-            0,
-            1,
-            0,
-            2
-          ],
-          [
-            0,
-            1,
-            0,
-            3
-          ],
-          [
-            0,
-            1,
-            1,
-            0
-          ],
-          [
-            0,
-            1,
-            1,
-            1
-          ],
-          [
-            0,
-            1,
-            1,
-            2
-          ],
-          [
-            0,
-            1,
-            1,
-            3
-          ],
-          [
-            0,
-            1,
-            2,
-            0
-          ],
-          [
-            0,
-            1,
-            2,
-            1
-          ],
-          [
-            0,
-            1,
-            2,
-            2
-          ],
-          [
-            0,
-            1,
-            2,
-            3
-          ],
-          [
-            0,
-            1,
-            3,
-            0
-          ],
-          [
-            0,
-            1,
-            3,
-            1
-          ],
-          [
-            0,
-            1,
-            3,
-            2
-          ],
-          [
-            0,
-            1,
-            3,
-            3
-          ],
-          [
-            0,
-            1,
-            4,
-            0
-          ],
-          [
-            0,
-            1,
-            4,
-            1
-          ],
-          [
-            0,
-            1,
-            4,
-            2
-          ],
-          [
-            0,
-            1,
-            4,
-            3
-          ],
-          [
-            0,
-            1,
-            5,
-            0
-          ],
-          [
-            0,
-            1,
-            5,
-            1
-          ],
-          [
-            0,
-            1,
-            5,
-            2
-          ],
-          [
-            0,
-            1,
-            5,
-            3
-          ],
-          [
-            0,
-            2,
-            0,
-            0
-          ],
-          [
-            0,
-            2,
-            0,
-            1
-          ]
-        ]
-      },
-      "qe_stagnation": {
-        "is_stagnant": false,
-        "current_qe": 0.009746708907186985,
-        "max_qe": 0.009746708907186985,
-        "slope": 0.0,
-        "severity": "none",
-        "n_epochs": 0
-      },
-      "neighborhood_crossing": {
-        "detected": false,
-        "te_initial_mean": 0.0,
-        "te_final_mean": 0.0,
-        "te_initial_std": 0.0,
-        "te_final_std": 0.0,
-        "std_ratio": 0.0,
-        "is_oscillating": false,
-        "is_growing": false,
-        "severity": "none",
-        "n_epochs": 0
-      },
-      "failure_indicators": [
-        "dead_neuron_rate_high_1.00",
-        "low_explained_variance_0.00",
-        "high_topological_error_1.00"
-      ],
-      "overall_health": "critical",
-      "n_failure_indicators": 3
-    },
-    "som_metric_history": {
-      "epoch": 6,
-      "qe_history": [
-        2.26032170758117e-05,
-        0.000569911440834403,
-        0.005070194136351347,
-        0.012031015940010548,
-        0.009746708907186985,
-        0.009746708907186985
-      ],
-      "te_history": [
-        0.880000002682209,
-        0.9140625,
-        0.984375,
-        1.0,
-        1.0,
-        1.0
-      ],
-      "kl_history": [
-        0.4445206419101888,
-        0.5460798889633685,
-        1.2844052385351477,
-        2.3798458777851814,
-        2.0229230836057637,
-        2.0229230836057637
-      ],
-      "var_explained_history": [
-        0.999451068570896,
-        0.928539884025705,
-        0.05886936612972116,
-        0.0,
-        0.0,
-        0.0
-      ],
-      "dead_neuron_rate_history": [
-        0.8900462962962963,
-        0.9456018518518519,
-        0.9884259259259259,
-        0.9976851851851852,
-        0.9976851851851852,
-        0.9976851851851852
-      ]
-    },
-    "final_state": {
-      "version": "V2-dynamic",
-      "n_hypotheses": 16,
-      "n_hypotheses_active": 16,
-      "max_n_hypotheses": 32,
-      "n_trials": 3,
-      "hyp_train_steps": 30,
-      "hyp_lr": 0.0001,
-      "delta_scale": 0.009999999776482582,
-      "n_generators": 32,
-      "punishment_count": 0,
-      "success_count": 0,
-      "training_ready": true,
-      "classifier_trained": false,
-      "ewc_reference_set": false,
-      "buffer_size": 128,
-      "total_hyp_steps_executed": 0,
-      "n_train_hyp_calls": 0,
-      "dynamic_adaptation": {
-        "loss_history_len": 0,
-        "loss_stats": {
-          "slope": 0.0,
-          "volatility": 0.0,
-          "mean": 0.0,
-          "n": 0
-        },
-        "punishment_rate": 0.0,
-        "punishment_window_size": 0,
-        "n_adaptations": 0,
-        "limits": {
-          "min_n_hypotheses": 4,
-          "max_n_hypotheses": 32,
-          "min_n_trials": 1,
-          "max_n_trials": 6,
-          "min_hyp_train_steps": 10,
-          "max_hyp_train_steps": 80
-        },
-        "last_5_adaptations": []
-      }
-    },
-    "auto_adjust_log": [
-      {
-        "step": 7,
-        "chunk_idx": 1,
-        "dataset": "dominguesm/restore-punctuation-ptbr-dataset",
-        "phase": "CONHECIMENTO",
-        "actions": [
-          "te_high_boost: σ+0.352 (te=0.880)",
-          "kl_high: kl=0.445 (covered by qe/te boosts)",
-          "conscience_auto_adjust: γ 0.103 → 0.099 (r=0.848)",
-          "boosts_applied: α₀_eff=0.5000, σ₀_eff=5.4837"
-        ],
-        "boosts": {
-          "alpha0_boost": 1.0,
-          "sigma0_boost": 1.3520000010728837
-        },
-        "n_actions": 4
-      },
-      {
-        "step": 14,
-        "chunk_idx": 2,
-        "dataset": "dominguesm/restore-punctuation-ptbr-dataset",
-        "phase": "CONHECIMENTO",
-        "actions": [
-          "te_high_boost: σ+0.366 (te=0.914)",
-          "kl_high: kl=0.546 (covered by qe/te boosts)",
-          "conscience_auto_adjust: γ 0.021 → 0.021 (r=0.856)",
-          "boosts_applied: α₀_eff=0.5000, σ₀_eff=8.8507"
-        ],
-        "boosts": {
-          "alpha0_boost": 1.0,
-          "sigma0_boost": 1.7176250010728837
-        },
-        "n_actions": 4
-      },
-      {
-        "step": 21,
-        "chunk_idx": 3,
-        "dataset": "dominguesm/restore-punctuation-ptbr-dataset",
-        "phase": "CONHECIMENTO",
-        "actions": [
-          "te_high_boost: σ+0.394 (te=0.984)",
-          "kl_high: kl=1.284 (covered by qe/te boosts)",
-          "ve_low_revive: n_revived=124 (ve=0.059)",
-          "conscience_auto_adjust: γ 0.021 → 0.021 (r=0.856)",
-          "boosts_applied: α₀_eff=0.5000, σ₀_eff=12.0000"
-        ],
-        "boosts": {
-          "alpha0_boost": 1.0,
-          "sigma0_boost": 2.0
-        },
-        "n_actions": 5
-      },
-      {
-        "step": 28,
-        "chunk_idx": 4,
-        "dataset": "dominguesm/restore-punctuation-ptbr-dataset",
-        "phase": "CONHECIMENTO",
-        "actions": [
-          "te_high_boost: σ+0.400 (te=1.000)",
-          "kl_high: kl=2.380 (covered by qe/te boosts)",
-          "ve_low_revive: n_revived=123 (ve=0.000)",
-          "conscience_auto_adjust: γ 0.020 → 0.020 (r=0.858)"
-        ],
-        "boosts": {
-          "alpha0_boost": 1.0,
-          "sigma0_boost": 2.0
-        },
-        "n_actions": 4
-      },
-      {
-        "step": 35,
-        "chunk_idx": 5,
-        "dataset": "dominguesm/restore-punctuation-ptbr-dataset",
-        "phase": "CONHECIMENTO",
-        "actions": [
-          "te_high_boost: σ+0.400 (te=1.000)",
-          "kl_high: kl=2.023 (covered by qe/te boosts)",
-          "ve_low_revive: n_revived=121 (ve=0.000)",
-          "conscience_auto_adjust: γ 0.020 → 0.020 (r=0.860)"
-        ],
-        "boosts": {
-          "alpha0_boost": 1.0,
-          "sigma0_boost": 2.0
-        },
-        "n_actions": 4
-      }
-    ],
-    "auto_adjust_state": {
-      "alpha0_boost": 1.0,
-      "sigma0_boost": 2.0,
-      "consecutive_collapse_point": 0,
-      "consecutive_qe_stagnant_severe": 0,
-      "consecutive_dead_rate_critical": 5,
-      "consecutive_nan_batches": 0,
-      "stop_training": true,
-      "stop_reason": "dead_rate_critical_consecutive=5",
-      "n_actions_total": 21,
-      "n_actions_logged": 5,
-      "recent_actions": [
-        {
-          "batch_idx": 1,
-          "actions": [
-            "te_high_boost: σ+0.352 (te=0.880)",
-            "kl_high: kl=0.445 (covered by qe/te boosts)",
-            "conscience_auto_adjust: γ 0.103 → 0.099 (r=0.848)",
-            "boosts_applied: α₀_eff=0.5000, σ₀_eff=5.4837"
-          ],
-          "metrics_snapshot": {
-            "qe": 2.26032170758117e-05,
-            "te": 0.880000002682209,
-            "kl": 0.4445206419101888,
-            "ve": 0.999451068570896,
-            "dead_rate": 0.8900462962962963,
-            "collapse_severity": "line",
-            "qe_stagnant": false,
-            "crossing_detected": false,
-            "nan_detected": false
-          },
-          "boosts": {
-            "alpha0_boost": 1.0,
-            "sigma0_boost": 1.3520000010728837
-          },
-          "stop_training": false
-        },
-        {
-          "batch_idx": 2,
-          "actions": [
-            "te_high_boost: σ+0.366 (te=0.914)",
-            "kl_high: kl=0.546 (covered by qe/te boosts)",
-            "conscience_auto_adjust: γ 0.021 → 0.021 (r=0.856)",
-            "boosts_applied: α₀_eff=0.5000, σ₀_eff=8.8507"
-          ],
-          "metrics_snapshot": {
-            "qe": 0.000569911440834403,
-            "te": 0.9140625,
-            "kl": 0.5460798889633685,
-            "ve": 0.928539884025705,
-            "dead_rate": 0.9456018518518519,
-            "collapse_severity": "line",
-            "qe_stagnant": false,
-            "crossing_detected": false,
-            "nan_detected": false
-          },
-          "boosts": {
-            "alpha0_boost": 1.0,
-            "sigma0_boost": 1.7176250010728837
-          },
-          "stop_training": false
-        },
-        {
-          "batch_idx": 3,
-          "actions": [
-            "te_high_boost: σ+0.394 (te=0.984)",
-            "kl_high: kl=1.284 (covered by qe/te boosts)",
-            "ve_low_revive: n_revived=124 (ve=0.059)",
-            "conscience_auto_adjust: γ 0.021 → 0.021 (r=0.856)",
-            "boosts_applied: α₀_eff=0.5000, σ₀_eff=12.0000"
-          ],
-          "metrics_snapshot": {
-            "qe": 0.005070194136351347,
-            "te": 0.984375,
-            "kl": 1.2844052385351477,
-            "ve": 0.05886936612972116,
-            "dead_rate": 0.9884259259259259,
-            "collapse_severity": "line",
-            "qe_stagnant": false,
-            "crossing_detected": false,
-            "nan_detected": false
-          },
-          "boosts": {
-            "alpha0_boost": 1.0,
-            "sigma0_boost": 2.0
-          },
-          "stop_training": false
-        },
-        {
-          "batch_idx": 4,
-          "actions": [
-            "te_high_boost: σ+0.400 (te=1.000)",
-            "kl_high: kl=2.380 (covered by qe/te boosts)",
-            "ve_low_revive: n_revived=123 (ve=0.000)",
-            "conscience_auto_adjust: γ 0.020 → 0.020 (r=0.858)"
-          ],
-          "metrics_snapshot": {
-            "qe": 0.012031015940010548,
-            "te": 1.0,
-            "kl": 2.3798458777851814,
-            "ve": 0.0,
-            "dead_rate": 0.9976851851851852,
-            "collapse_severity": "none",
-            "qe_stagnant": false,
-            "crossing_detected": false,
-            "nan_detected": false
-          },
-          "boosts": {
-            "alpha0_boost": 1.0,
-            "sigma0_boost": 2.0
-          },
-          "stop_training": false
-        },
-        {
-          "batch_idx": 5,
-          "actions": [
-            "te_high_boost: σ+0.400 (te=1.000)",
-            "kl_high: kl=2.023 (covered by qe/te boosts)",
-            "ve_low_revive: n_revived=121 (ve=0.000)",
-            "conscience_auto_adjust: γ 0.020 → 0.020 (r=0.860)"
-          ],
-          "metrics_snapshot": {
-            "qe": 0.009746708907186985,
-            "te": 1.0,
-            "kl": 2.0229230836057637,
-            "ve": 0.0,
-            "dead_rate": 0.9976851851851852,
-            "collapse_severity": "none",
-            "qe_stagnant": false,
-            "crossing_detected": false,
-            "nan_detected": false
-          },
-          "boosts": {
-            "alpha0_boost": 1.0,
-            "sigma0_boost": 2.0
-          },
-          "stop_training": true
-        }
-      ]
-    },
-    "stop_training_reason": "dead_rate_critical_consecutive=5"
-  }
-}

v6_5_v2_predict_fix_eval.json

@@ -1,84 +0,0 @@
-{
-  "evaluation": "predict_fix_v65_v2",
-  "user_requirement": "os strings 'gato' e 'cachorro' são fixos quando deveriam ser extrações variáveis e flexíveis de rótulos proveniente de dados dos datasets anteriormente treinados",
-  "n_test_queries": 6,
-  "results": [
-    {
-      "query": "o gato dorme na cama",
-      "prediction": "punctuated",
-      "probability": 1.0,
-      "is_gato_hardcoded": false,
-      "is_cachorro_hardcoded": false,
-      "is_registry_label": true,
-      "is_default_label": false
-    },
-    {
-      "query": "calcule dois mais dois",
-      "prediction": "punctuated",
-      "probability": 1.0,
-      "is_gato_hardcoded": false,
-      "is_cachorro_hardcoded": false,
-      "is_registry_label": true,
-      "is_default_label": false
-    },
-    {
-      "query": "qual é a capital do brasil",
-      "prediction": "punctuated",
-      "probability": 1.0,
-      "is_gato_hardcoded": false,
-      "is_cachorro_hardcoded": false,
-      "is_registry_label": true,
-      "is_default_label": false
-    },
-    {
-      "query": "explique o que é uma rede neural",
-      "prediction": "punctuated",
-      "probability": 1.0,
-      "is_gato_hardcoded": false,
-      "is_cachorro_hardcoded": false,
-      "is_registry_label": true,
-      "is_default_label": false
-    },
-    {
-      "query": "olá como você está",
-      "prediction": "punctuated",
-      "probability": 1.0,
-      "is_gato_hardcoded": false,
-      "is_cachorro_hardcoded": false,
-      "is_registry_label": true,
-      "is_default_label": false
-    },
-    {
-      "query": "traduza hello para portugues",
-      "prediction": "punctuated",
-      "probability": 1.0,
-      "is_gato_hardcoded": false,
-      "is_cachorro_hardcoded": false,
-      "is_registry_label": true,
-      "is_default_label": false
-    }
-  ],
-  "per_dataset_results": [
-    {
-      "dataset": "dominguesm/restore-punctuation-ptbr-dataset",
-      "expected_labels": [
-        "unpunctuated",
-        "punctuated"
-      ],
-      "prediction": "punctuated",
-      "valid_for_dataset": true
-    }
-  ],
-  "summary": {
-    "n_returns_gato": 0,
-    "n_returns_cachorro": 0,
-    "n_returns_registry_label": 6,
-    "n_returns_default_label": 0,
-    "hardcoded_bug_present": false,
-    "predict_fix_verified": true
-  },
-  "quality_assessment": {
-    "fix_status": "PASS",
-    "note": "predict() não retorna mais 'gato'/'cachorro' fixos — usa label_registry dinâmico."
-  }
-}

v6_5_v2_punicão_report.json

@@ -1,1199 +0,0 @@
-{
-  "version": "V6.5-V2-dynamic-punicão",
-  "timestamp": "2026-08-08T21:32:29.242618",
-  "loaded_from_fase1": "/home/z/my-project/BiGRU_T_version/v6_5_v2_model_states.pt",
-  "fase1_total_samples": 3300,
-  "fase2_total_samples": 1000,
-  "fase2_meta_atingida": true,
-  "fase2_elapsed_s": 135.41348695755005,
-  "punishment_events": [
-    {
-      "step": 0,
-      "action": "train_hypotheses_and_activate",
-      "accuracy": 0.5,
-      "punishment_count": 1,
-      "cycle_completed": false
-    },
-    {
-      "step": 1,
-      "action": "apply_best_delta_and_consolidate",
-      "accuracy": 0.625,
-      "punishment_count": 0,
-      "cycle_completed": true
-    },
-    {
-      "step": 2,
-      "action": "train_hypotheses_and_activate",
-      "accuracy": 0.5416666666666666,
-      "punishment_count": 1,
-      "cycle_completed": false
-    },
-    {
-      "step": 3,
-      "action": "apply_best_delta_and_consolidate",
-      "accuracy": 0.546875,
-      "punishment_count": 0,
-      "cycle_completed": true
-    },
-    {
-      "step": 4,
-      "action": "train_hypotheses_and_activate",
-      "accuracy": 0.5625,
-      "punishment_count": 1,
-      "cycle_completed": false
-    },
-    {
-      "step": 5,
-      "action": "apply_best_delta_and_consolidate",
-      "accuracy": 0.5520833333333334,
-      "punishment_count": 0,
-      "cycle_completed": true
-    },
-    {
-      "step": 7,
-      "action": "train_hypotheses_and_activate",
-      "accuracy": 0.5517241379310345,
-      "punishment_count": 1,
-      "cycle_completed": false
-    },
-    {
-      "step": 8,
-      "action": "apply_best_delta_and_consolidate",
-      "accuracy": 0.5454545454545454,
-      "punishment_count": 0,
-      "cycle_completed": true
-    },
-    {
-      "step": 9,
-      "action": "train_hypotheses_and_activate",
-      "accuracy": 0.5337837837837838,
-      "punishment_count": 1,
-      "cycle_completed": false
-    },
-    {
-      "step": 10,
-      "action": "apply_best_delta_and_consolidate",
-      "accuracy": 0.5426829268292683,
-      "punishment_count": 0,
-      "cycle_completed": true
-    },
-    {
-      "step": 11,
-      "action": "train_hypotheses_and_activate",
-      "accuracy": 0.5388888888888889,
-      "punishment_count": 1,
-      "cycle_completed": false
-    },
-    {
-      "step": 12,
-      "action": "apply_best_delta_and_consolidate",
-      "accuracy": 0.5204081632653061,
-      "punishment_count": 0,
-      "cycle_completed": true
-    },
-    {
-      "step": 14,
-      "action": "train_hypotheses_and_activate",
-      "accuracy": 0.5092592592592593,
-      "punishment_count": 1,
-      "cycle_completed": false
-    },
-    {
-      "step": 15,
-      "action": "apply_best_delta_and_consolidate",
-      "accuracy": 0.5043103448275862,
-      "punishment_count": 0,
-      "cycle_completed": true
-    },
-    {
-      "step": 16,
-      "action": "train_hypotheses_and_activate",
-      "accuracy": 0.5080645161290323,
-      "punishment_count": 1,
-      "cycle_completed": false
-    },
-    {
-      "step": 17,
-      "action": "apply_best_delta_and_consolidate",
-      "accuracy": 0.5,
-      "punishment_count": 0,
-      "cycle_completed": true
-    },
-    {
-      "step": 18,
-      "action": "train_hypotheses_and_activate",
-      "accuracy": 0.5183823529411765,
-      "punishment_count": 1,
-      "cycle_completed": false
-    },
-    {
-      "step": 19,
-      "action": "apply_best_delta_and_consolidate",
-      "accuracy": 0.5173611111111112,
-      "punishment_count": 0,
-      "cycle_completed": true
-    },
-    {
-      "step": 21,
-      "action": "train_hypotheses_and_activate",
-      "accuracy": 0.5110294117647058,
-      "punishment_count": 1,
-      "cycle_completed": false
-    },
-    {
-      "step": 22,
-      "action": "apply_best_delta_and_consolidate",
-      "accuracy": 0.5138888888888888,
-      "punishment_count": 0,
-      "cycle_completed": true
-    },
-    {
-      "step": 23,
-      "action": "train_hypotheses_and_activate",
-      "accuracy": 0.5036764705882353,
-      "punishment_count": 1,
-      "cycle_completed": false
-    },
-    {
-      "step": 24,
-      "action": "apply_best_delta_and_consolidate",
-      "accuracy": 0.4930555555555556,
-      "punishment_count": 0,
-      "cycle_completed": true
-    },
-    {
-      "step": 25,
-      "action": "train_hypotheses_and_activate",
-      "accuracy": 0.4889705882352941,
-      "punishment_count": 1,
-      "cycle_completed": false
-    },
-    {
-      "step": 26,
-      "action": "apply_best_delta_and_consolidate",
-      "accuracy": 0.4756944444444444,
-      "punishment_count": 0,
-      "cycle_completed": true
-    },
-    {
-      "step": 28,
-      "action": "train_hypotheses_and_activate",
-      "accuracy": 0.47794117647058826,
-      "punishment_count": 1,
-      "cycle_completed": false
-    },
-    {
-      "step": 29,
-      "action": "apply_best_delta_and_consolidate",
-      "accuracy": 0.4895833333333333,
-      "punishment_count": 0,
-      "cycle_completed": true
-    },
-    {
-      "step": 30,
-      "action": "train_hypotheses_and_activate",
-      "accuracy": 0.4742647058823529,
-      "punishment_count": 1,
-      "cycle_completed": false
-    },
-    {
-      "step": 31,
-      "action": "apply_best_delta_and_consolidate",
-      "accuracy": 0.4826388888888889,
-      "punishment_count": 0,
-      "cycle_completed": true
-    },
-    {
-      "step": 32,
-      "action": "train_hypotheses_and_activate",
-      "accuracy": 0.5073529411764706,
-      "punishment_count": 1,
-      "cycle_completed": false
-    },
-    {
-      "step": 33,
-      "action": "apply_best_delta_and_consolidate",
-      "accuracy": 0.5138888888888888,
-      "punishment_count": 0,
-      "cycle_completed": true
-    },
-    {
-      "step": 35,
-      "action": "train_hypotheses_and_activate",
-      "accuracy": 0.5110294117647058,
-      "punishment_count": 1,
-      "cycle_completed": false
-    },
-    {
-      "step": 36,
-      "action": "apply_best_delta_and_consolidate",
-      "accuracy": 0.5104166666666666,
-      "punishment_count": 0,
-      "cycle_completed": true
-    },
-    {
-      "step": 37,
-      "action": "train_hypotheses_and_activate",
-      "accuracy": 0.47794117647058826,
-      "punishment_count": 1,
-      "cycle_completed": false
-    },
-    {
-      "step": 38,
-      "action": "apply_best_delta_and_consolidate",
-      "accuracy": 0.4791666666666667,
-      "punishment_count": 0,
-      "cycle_completed": true
-    },
-    {
-      "step": 39,
-      "action": "train_hypotheses_and_activate",
-      "accuracy": 0.47058823529411764,
-      "punishment_count": 1,
-      "cycle_completed": false
-    },
-    {
-      "step": 40,
-      "action": "apply_best_delta_and_consolidate",
-      "accuracy": 0.4930555555555556,
-      "punishment_count": 0,
-      "cycle_completed": true
-    },
-    {
-      "step": 42,
-      "action": "train_hypotheses_and_activate",
-      "accuracy": 0.4889705882352941,
-      "punishment_count": 1,
-      "cycle_completed": false
-    },
-    {
-      "step": 43,
-      "action": "apply_best_delta_and_consolidate",
-      "accuracy": 0.4965277777777778,
-      "punishment_count": 0,
-      "cycle_completed": true
-    },
-    {
-      "step": 44,
-      "action": "train_hypotheses_and_activate",
-      "accuracy": 0.5110294117647058,
-      "punishment_count": 1,
-      "cycle_completed": false
-    },
-    {
-      "step": 45,
-      "action": "apply_best_delta_and_consolidate",
-      "accuracy": 0.5138888888888888,
-      "punishment_count": 0,
-      "cycle_completed": true
-    },
-    {
-      "step": 46,
-      "action": "train_hypotheses_and_activate",
-      "accuracy": 0.5367647058823529,
-      "punishment_count": 1,
-      "cycle_completed": false
-    },
-    {
-      "step": 47,
-      "action": "apply_best_delta_and_consolidate",
-      "accuracy": 0.5208333333333334,
-      "punishment_count": 0,
-      "cycle_completed": true
-    },
-    {
-      "step": 49,
-      "action": "train_hypotheses_and_activate",
-      "accuracy": 0.5110294117647058,
-      "punishment_count": 1,
-      "cycle_completed": false
-    },
-    {
-      "step": 50,
-      "action": "apply_best_delta_and_consolidate",
-      "accuracy": 0.5208333333333334,
-      "punishment_count": 0,
-      "cycle_completed": true
-    },
-    {
-      "step": 51,
-      "action": "train_hypotheses_and_activate",
-      "accuracy": 0.5183823529411765,
-      "punishment_count": 1,
-      "cycle_completed": false
-    },
-    {
-      "step": 52,
-      "action": "apply_best_delta_and_consolidate",
-      "accuracy": 0.5173611111111112,
-      "punishment_count": 0,
-      "cycle_completed": true
-    },
-    {
-      "step": 53,
-      "action": "train_hypotheses_and_activate",
-      "accuracy": 0.5073529411764706,
-      "punishment_count": 1,
-      "cycle_completed": false
-    },
-    {
-      "step": 54,
-      "action": "apply_best_delta_and_consolidate",
-      "accuracy": 0.5104166666666666,
-      "punishment_count": 0,
-      "cycle_completed": true
-    },
-    {
-      "step": 56,
-      "action": "train_hypotheses_and_activate",
-      "accuracy": 0.5110294117647058,
-      "punishment_count": 1,
-      "cycle_completed": false
-    },
-    {
-      "step": 57,
-      "action": "apply_best_delta_and_consolidate",
-      "accuracy": 0.4965277777777778,
-      "punishment_count": 0,
-      "cycle_completed": true
-    },
-    {
-      "step": 58,
-      "action": "train_hypotheses_and_activate",
-      "accuracy": 0.5147058823529411,
-      "punishment_count": 1,
-      "cycle_completed": false
-    },
-    {
-      "step": 59,
-      "action": "apply_best_delta_and_consolidate",
-      "accuracy": 0.5104166666666666,
-      "punishment_count": 0,
-      "cycle_completed": true
-    },
-    {
-      "step": 60,
-      "action": "train_hypotheses_and_activate",
-      "accuracy": 0.4963235294117647,
-      "punishment_count": 1,
-      "cycle_completed": false
-    },
-    {
-      "step": 61,
-      "action": "apply_best_delta_and_consolidate",
-      "accuracy": 0.4895833333333333,
-      "punishment_count": 0,
-      "cycle_completed": true
-    },
-    {
-      "step": 63,
-      "action": "train_hypotheses_and_activate",
-      "accuracy": 0.46691176470588236,
-      "punishment_count": 1,
-      "cycle_completed": false
-    },
-    {
-      "step": 64,
-      "action": "apply_best_delta_and_consolidate",
-      "accuracy": 0.4722222222222222,
-      "punishment_count": 0,
-      "cycle_completed": true
-    },
-    {
-      "step": 65,
-      "action": "train_hypotheses_and_activate",
-      "accuracy": 0.45588235294117646,
-      "punishment_count": 1,
-      "cycle_completed": false
-    },
-    {
-      "step": 66,
-      "action": "apply_best_delta_and_consolidate",
-      "accuracy": 0.4583333333333333,
-      "punishment_count": 0,
-      "cycle_completed": true
-    },
-    {
-      "step": 67,
-      "action": "train_hypotheses_and_activate",
-      "accuracy": 0.4632352941176471,
-      "punishment_count": 1,
-      "cycle_completed": false
-    },
-    {
-      "step": 68,
-      "action": "apply_best_delta_and_consolidate",
-      "accuracy": 0.4722222222222222,
-      "punishment_count": 0,
-      "cycle_completed": true
-    }
-  ],
-  "hypotheses_trainings": [
-    {
-      "step": 0,
-      "loss_initial": NaN,
-      "loss_final": NaN,
-      "delta_scale": 0.008179070428013802
-    },
-    {
-      "step": 2,
-      "loss_initial": NaN,
-      "loss_final": NaN,
-      "delta_scale": 0.00668971985578537
-    },
-    {
-      "step": 4,
-      "loss_initial": NaN,
-      "loss_final": NaN,
-      "delta_scale": 0.005471568554639816
-    },
-    {
-      "step": 7,
-      "loss_initial": NaN,
-      "loss_final": NaN,
-      "delta_scale": 0.00447523407638073
-    },
-    {
-      "step": 9,
-      "loss_initial": NaN,
-      "loss_final": NaN,
-      "delta_scale": 0.003660326125100255
-    },
-    {
-      "step": 11,
-      "loss_initial": NaN,
-      "loss_final": NaN,
-      "delta_scale": 0.0029938064981251955
-    },
-    {
-      "step": 14,
-      "loss_initial": NaN,
-      "loss_final": NaN,
-      "delta_scale": 0.002448655664920807
-    },
-    {
-      "step": 16,
-      "loss_initial": NaN,
-      "loss_final": NaN,
-      "delta_scale": 0.0020027728751301765
-    },
-    {
-      "step": 18,
-      "loss_initial": NaN,
-      "loss_final": NaN,
-      "delta_scale": 0.0016380823217332363
-    },
-    {
-      "step": 21,
-      "loss_initial": NaN,
-      "loss_final": NaN,
-      "delta_scale": 0.0013397991424426436
-    },
-    {
-      "step": 23,
-      "loss_initial": NaN,
-      "loss_final": NaN,
-      "delta_scale": 0.0010958312777802348
-    },
-    {
-      "step": 25,
-      "loss_initial": NaN,
-      "loss_final": NaN,
-      "delta_scale": 0.0010000000474974513
-    },
-    {
-      "step": 28,
-      "loss_initial": NaN,
-      "loss_final": NaN,
-      "delta_scale": 0.0010000000474974513
-    },
-    {
-      "step": 30,
-      "loss_initial": NaN,
-      "loss_final": NaN,
-      "delta_scale": 0.0010000000474974513
-    },
-    {
-      "step": 32,
-      "loss_initial": NaN,
-      "loss_final": NaN,
-      "delta_scale": 0.0010000000474974513
-    },
-    {
-      "step": 35,
-      "loss_initial": NaN,
-      "loss_final": NaN,
-      "delta_scale": 0.0010000000474974513
-    },
-    {
-      "step": 37,
-      "loss_initial": NaN,
-      "loss_final": NaN,
-      "delta_scale": 0.0010000000474974513
-    },
-    {
-      "step": 39,
-      "loss_initial": NaN,
-      "loss_final": NaN,
-      "delta_scale": 0.0010000000474974513
-    },
-    {
-      "step": 42,
-      "loss_initial": NaN,
-      "loss_final": NaN,
-      "delta_scale": 0.0010000000474974513
-    },
-    {
-      "step": 44,
-      "loss_initial": NaN,
-      "loss_final": NaN,
-      "delta_scale": 0.0010000000474974513
-    },
-    {
-      "step": 46,
-      "loss_initial": NaN,
-      "loss_final": NaN,
-      "delta_scale": 0.0010000000474974513
-    },
-    {
-      "step": 49,
-      "loss_initial": NaN,
-      "loss_final": NaN,
-      "delta_scale": 0.0010000000474974513
-    },
-    {
-      "step": 51,
-      "loss_initial": NaN,
-      "loss_final": NaN,
-      "delta_scale": 0.0010000000474974513
-    },
-    {
-      "step": 53,
-      "loss_initial": NaN,
-      "loss_final": NaN,
-      "delta_scale": 0.0010000000474974513
-    },
-    {
-      "step": 56,
-      "loss_initial": NaN,
-      "loss_final": NaN,
-      "delta_scale": 0.0010000000474974513
-    },
-    {
-      "step": 58,
-      "loss_initial": NaN,
-      "loss_final": NaN,
-      "delta_scale": 0.0010000000474974513
-    },
-    {
-      "step": 60,
-      "loss_initial": NaN,
-      "loss_final": NaN,
-      "delta_scale": 0.0010000000474974513
-    },
-    {
-      "step": 63,
-      "loss_initial": NaN,
-      "loss_final": NaN,
-      "delta_scale": 0.0010000000474974513
-    },
-    {
-      "step": 65,
-      "loss_initial": NaN,
-      "loss_final": NaN,
-      "delta_scale": 0.0010000000474974513
-    },
-    {
-      "step": 67,
-      "loss_initial": NaN,
-      "loss_final": NaN,
-      "delta_scale": 0.0010000000474974513
-    }
-  ],
-  "delta_applications": [
-    {
-      "step": 1,
-      "acc_before": 0.625,
-      "acc_after": 0.625,
-      "delta_norm": NaN,
-      "ewc_reference_set": true
-    },
-    {
-      "step": 3,
-      "acc_before": 0.546875,
-      "acc_after": 0.546875,
-      "delta_norm": NaN,
-      "ewc_reference_set": true
-    },
-    {
-      "step": 5,
-      "acc_before": 0.5520833333333334,
-      "acc_after": 0.5520833333333334,
-      "delta_norm": NaN,
-      "ewc_reference_set": true
-    },
-    {
-      "step": 8,
-      "acc_before": 0.5454545454545454,
-      "acc_after": 0.5454545454545454,
-      "delta_norm": NaN,
-      "ewc_reference_set": true
-    },
-    {
-      "step": 10,
-      "acc_before": 0.5426829268292683,
-      "acc_after": 0.5426829268292683,
-      "delta_norm": NaN,
-      "ewc_reference_set": true
-    },
-    {
-      "step": 12,
-      "acc_before": 0.5204081632653061,
-      "acc_after": 0.5204081632653061,
-      "delta_norm": NaN,
-      "ewc_reference_set": true
-    },
-    {
-      "step": 15,
-      "acc_before": 0.5043103448275862,
-      "acc_after": 0.5043103448275862,
-      "delta_norm": NaN,
-      "ewc_reference_set": true
-    },
-    {
-      "step": 17,
-      "acc_before": 0.5,
-      "acc_after": 0.5,
-      "delta_norm": NaN,
-      "ewc_reference_set": true
-    },
-    {
-      "step": 19,
-      "acc_before": 0.5173611111111112,
-      "acc_after": 0.5173611111111112,
-      "delta_norm": NaN,
-      "ewc_reference_set": true
-    },
-    {
-      "step": 22,
-      "acc_before": 0.5138888888888888,
-      "acc_after": 0.5138888888888888,
-      "delta_norm": NaN,
-      "ewc_reference_set": true
-    },
-    {
-      "step": 24,
-      "acc_before": 0.4930555555555556,
-      "acc_after": 0.4930555555555556,
-      "delta_norm": NaN,
-      "ewc_reference_set": true
-    },
-    {
-      "step": 26,
-      "acc_before": 0.4756944444444444,
-      "acc_after": 0.4756944444444444,
-      "delta_norm": NaN,
-      "ewc_reference_set": true
-    },
-    {
-      "step": 29,
-      "acc_before": 0.4895833333333333,
-      "acc_after": 0.4895833333333333,
-      "delta_norm": NaN,
-      "ewc_reference_set": true
-    },
-    {
-      "step": 31,
-      "acc_before": 0.4826388888888889,
-      "acc_after": 0.4826388888888889,
-      "delta_norm": NaN,
-      "ewc_reference_set": true
-    },
-    {
-      "step": 33,
-      "acc_before": 0.5138888888888888,
-      "acc_after": 0.5138888888888888,
-      "delta_norm": NaN,
-      "ewc_reference_set": true
-    },
-    {
-      "step": 36,
-      "acc_before": 0.5104166666666666,
-      "acc_after": 0.5104166666666666,
-      "delta_norm": NaN,
-      "ewc_reference_set": true
-    },
-    {
-      "step": 38,
-      "acc_before": 0.4791666666666667,
-      "acc_after": 0.4791666666666667,
-      "delta_norm": NaN,
-      "ewc_reference_set": true
-    },
-    {
-      "step": 40,
-      "acc_before": 0.4930555555555556,
-      "acc_after": 0.4930555555555556,
-      "delta_norm": NaN,
-      "ewc_reference_set": true
-    },
-    {
-      "step": 43,
-      "acc_before": 0.4965277777777778,
-      "acc_after": 0.4965277777777778,
-      "delta_norm": NaN,
-      "ewc_reference_set": true
-    },
-    {
-      "step": 45,
-      "acc_before": 0.5138888888888888,
-      "acc_after": 0.5138888888888888,
-      "delta_norm": NaN,
-      "ewc_reference_set": true
-    },
-    {
-      "step": 47,
-      "acc_before": 0.5208333333333334,
-      "acc_after": 0.5208333333333334,
-      "delta_norm": NaN,
-      "ewc_reference_set": true
-    },
-    {
-      "step": 50,
-      "acc_before": 0.5208333333333334,
-      "acc_after": 0.5208333333333334,
-      "delta_norm": NaN,
-      "ewc_reference_set": true
-    },
-    {
-      "step": 52,
-      "acc_before": 0.5173611111111112,
-      "acc_after": 0.5173611111111112,
-      "delta_norm": NaN,
-      "ewc_reference_set": true
-    },
-    {
-      "step": 54,
-      "acc_before": 0.5104166666666666,
-      "acc_after": 0.5104166666666666,
-      "delta_norm": NaN,
-      "ewc_reference_set": true
-    },
-    {
-      "step": 57,
-      "acc_before": 0.4965277777777778,
-      "acc_after": 0.4965277777777778,
-      "delta_norm": NaN,
-      "ewc_reference_set": true
-    },
-    {
-      "step": 59,
-      "acc_before": 0.5104166666666666,
-      "acc_after": 0.5104166666666666,
-      "delta_norm": NaN,
-      "ewc_reference_set": true
-    },
-    {
-      "step": 61,
-      "acc_before": 0.4895833333333333,
-      "acc_after": 0.4895833333333333,
-      "delta_norm": NaN,
-      "ewc_reference_set": true
-    },
-    {
-      "step": 64,
-      "acc_before": 0.4722222222222222,
-      "acc_after": 0.4722222222222222,
-      "delta_norm": NaN,
-      "ewc_reference_set": true
-    },
-    {
-      "step": 66,
-      "acc_before": 0.4583333333333333,
-      "acc_after": 0.4583333333333333,
-      "delta_norm": NaN,
-      "ewc_reference_set": true
-    },
-    {
-      "step": 68,
-      "acc_before": 0.4722222222222222,
-      "acc_after": 0.4722222222222222,
-      "delta_norm": NaN,
-      "ewc_reference_set": true
-    }
-  ],
-  "accuracies_log": [
-    {
-      "step": 40,
-      "accuracy": 0.48046875,
-      "action": "apply_best_delta_and_consolidate",
-      "punishment_count": 0,
-      "buffer_size": 256
-    },
-    {
-      "step": 41,
-      "accuracy": 0.4807692307692308,
-      "action": "none",
-      "punishment_count": 0,
-      "buffer_size": 260
-    },
-    {
-      "step": 42,
-      "accuracy": 0.4889705882352941,
-      "action": "train_hypotheses_and_activate",
-      "punishment_count": 1,
-      "buffer_size": 272
-    },
-    {
-      "step": 43,
-      "accuracy": 0.5,
-      "action": "apply_best_delta_and_consolidate",
-      "punishment_count": 0,
-      "buffer_size": 256
-    },
-    {
-      "step": 44,
-      "accuracy": 0.5110294117647058,
-      "action": "train_hypotheses_and_activate",
-      "punishment_count": 1,
-      "buffer_size": 272
-    },
-    {
-      "step": 45,
-      "accuracy": 0.53515625,
-      "action": "apply_best_delta_and_consolidate",
-      "punishment_count": 0,
-      "buffer_size": 256
-    },
-    {
-      "step": 46,
-      "accuracy": 0.5367647058823529,
-      "action": "train_hypotheses_and_activate",
-      "punishment_count": 1,
-      "buffer_size": 272
-    },
-    {
-      "step": 47,
-      "accuracy": 0.51171875,
-      "action": "apply_best_delta_and_consolidate",
-      "punishment_count": 0,
-      "buffer_size": 256
-    },
-    {
-      "step": 48,
-      "accuracy": 0.5076923076923077,
-      "action": "none",
-      "punishment_count": 0,
-      "buffer_size": 260
-    },
-    {
-      "step": 49,
-      "accuracy": 0.5110294117647058,
-      "action": "train_hypotheses_and_activate",
-      "punishment_count": 1,
-      "buffer_size": 272
-    },
-    {
-      "step": 50,
-      "accuracy": 0.515625,
-      "action": "apply_best_delta_and_consolidate",
-      "punishment_count": 0,
-      "buffer_size": 256
-    },
-    {
-      "step": 51,
-      "accuracy": 0.5183823529411765,
-      "action": "train_hypotheses_and_activate",
-      "punishment_count": 1,
-      "buffer_size": 272
-    },
-    {
-      "step": 52,
-      "accuracy": 0.51953125,
-      "action": "apply_best_delta_and_consolidate",
-      "punishment_count": 0,
-      "buffer_size": 256
-    },
-    {
-      "step": 53,
-      "accuracy": 0.5073529411764706,
-      "action": "train_hypotheses_and_activate",
-      "punishment_count": 1,
-      "buffer_size": 272
-    },
-    {
-      "step": 54,
-      "accuracy": 0.5234375,
-      "action": "apply_best_delta_and_consolidate",
-      "punishment_count": 0,
-      "buffer_size": 256
-    },
-    {
-      "step": 55,
-      "accuracy": 0.5230769230769231,
-      "action": "none",
-      "punishment_count": 0,
-      "buffer_size": 260
-    },
-    {
-      "step": 56,
-      "accuracy": 0.5110294117647058,
-      "action": "train_hypotheses_and_activate",
-      "punishment_count": 1,
-      "buffer_size": 272
-    },
-    {
-      "step": 57,
-      "accuracy": 0.51171875,
-      "action": "apply_best_delta_and_consolidate",
-      "punishment_count": 0,
-      "buffer_size": 256
-    },
-    {
-      "step": 58,
-      "accuracy": 0.5147058823529411,
-      "action": "train_hypotheses_and_activate",
-      "punishment_count": 1,
-      "buffer_size": 272
-    },
-    {
-      "step": 59,
-      "accuracy": 0.5,
-      "action": "apply_best_delta_and_consolidate",
-      "punishment_count": 0,
-      "buffer_size": 256
-    },
-    {
-      "step": 60,
-      "accuracy": 0.4963235294117647,
-      "action": "train_hypotheses_and_activate",
-      "punishment_count": 1,
-      "buffer_size": 272
-    },
-    {
-      "step": 61,
-      "accuracy": 0.45703125,
-      "action": "apply_best_delta_and_consolidate",
-      "punishment_count": 0,
-      "buffer_size": 256
-    },
-    {
-      "step": 62,
-      "accuracy": 0.4576923076923077,
-      "action": "none",
-      "punishment_count": 0,
-      "buffer_size": 260
-    },
-    {
-      "step": 63,
-      "accuracy": 0.46691176470588236,
-      "action": "train_hypotheses_and_activate",
-      "punishment_count": 1,
-      "buffer_size": 272
-    },
-    {
-      "step": 64,
-      "accuracy": 0.46484375,
-      "action": "apply_best_delta_and_consolidate",
-      "punishment_count": 0,
-      "buffer_size": 256
-    },
-    {
-      "step": 65,
-      "accuracy": 0.45588235294117646,
-      "action": "train_hypotheses_and_activate",
-      "punishment_count": 1,
-      "buffer_size": 272
-    },
-    {
-      "step": 66,
-      "accuracy": 0.4609375,
-      "action": "apply_best_delta_and_consolidate",
-      "punishment_count": 0,
-      "buffer_size": 256
-    },
-    {
-      "step": 67,
-      "accuracy": 0.4632352941176471,
-      "action": "train_hypotheses_and_activate",
-      "punishment_count": 1,
-      "buffer_size": 272
-    },
-    {
-      "step": 68,
-      "accuracy": 0.46484375,
-      "action": "apply_best_delta_and_consolidate",
-      "punishment_count": 0,
-      "buffer_size": 256
-    },
-    {
-      "step": 69,
-      "accuracy": 0.4653846153846154,
-      "action": "none",
-      "punishment_count": 0,
-      "buffer_size": 260
-    }
-  ],
-  "final_v2_metrics": {
-    "version": "V2-dynamic",
-    "n_hypotheses": 8,
-    "n_hypotheses_active": 8,
-    "max_n_hypotheses": 16,
-    "n_trials": 6,
-    "hyp_train_steps": 20,
-    "hyp_lr": 0.0001,
-    "delta_scale": 0.0010000000474974513,
-    "n_generators": 16,
-    "punishment_count": 0,
-    "success_count": 0,
-    "training_ready": false,
-    "classifier_trained": true,
-    "ewc_reference_set": true,
-    "buffer_size": 256,
-    "dynamic_adaptation": {
-      "loss_history_len": 8,
-      "loss_stats": {
-        "slope": NaN,
-        "volatility": NaN,
-        "mean": NaN,
-        "std": NaN,
-        "n": 8
-      },
-      "punishment_rate": 1.0,
-      "punishment_window_size": 12,
-      "n_adaptations": 60,
-      "limits": {
-        "min_n_hypotheses": 4,
-        "max_n_hypotheses": 16,
-        "min_n_trials": 1,
-        "max_n_trials": 6,
-        "min_hyp_train_steps": 5,
-        "max_hyp_train_steps": 60
-      },
-      "last_5_adaptations": [
-        {
-          "trigger": "punishment",
-          "step": 55,
-          "before": {
-            "n_hypotheses": 8,
-            "n_trials": 6,
-            "hyp_train_steps": 20
-          },
-          "after": {
-            "n_hypotheses": 8,
-            "n_trials": 6,
-            "hyp_train_steps": 20
-          },
-          "adapted": false,
-          "rules_fired": [],
-          "loss_stats": {
-            "slope": NaN,
-            "volatility": NaN,
-            "mean": NaN,
-            "std": NaN,
-            "n": 8
-          },
-          "punishment_rate": 1.0
-        },
-        {
-          "trigger": "auto",
-          "step": 56,
-          "before": {
-            "n_hypotheses": 8,
-            "n_trials": 6,
-            "hyp_train_steps": 20
-          },
-          "after": {
-            "n_hypotheses": 8,
-            "n_trials": 6,
-            "hyp_train_steps": 20
-          },
-          "adapted": false,
-          "rules_fired": [],
-          "loss_stats": {
-            "slope": NaN,
-            "volatility": NaN,
-            "mean": NaN,
-            "std": NaN,
-            "n": 8
-          },
-          "punishment_rate": 1.0
-        },
-        {
-          "trigger": "punishment",
-          "step": 57,
-          "before": {
-            "n_hypotheses": 8,
-            "n_trials": 6,
-            "hyp_train_steps": 20
-          },
-          "after": {
-            "n_hypotheses": 8,
-            "n_trials": 6,
-            "hyp_train_steps": 20
-          },
-          "adapted": false,
-          "rules_fired": [],
-          "loss_stats": {
-            "slope": NaN,
-            "volatility": NaN,
-            "mean": NaN,
-            "std": NaN,
-            "n": 8
-          },
-          "punishment_rate": 1.0
-        },
-        {
-          "trigger": "auto",
-          "step": 58,
-          "before": {
-            "n_hypotheses": 8,
-            "n_trials": 6,
-            "hyp_train_steps": 20
-          },
-          "after": {
-            "n_hypotheses": 8,
-            "n_trials": 6,
-            "hyp_train_steps": 20
-          },
-          "adapted": false,
-          "rules_fired": [],
-          "loss_stats": {
-            "slope": NaN,
-            "volatility": NaN,
-            "mean": NaN,
-            "std": NaN,
-            "n": 8
-          },
-          "punishment_rate": 1.0
-        },
-        {
-          "trigger": "punishment",
-          "step": 59,
-          "before": {
-            "n_hypotheses": 8,
-            "n_trials": 6,
-            "hyp_train_steps": 20
-          },
-          "after": {
-            "n_hypotheses": 8,
-            "n_trials": 6,
-            "hyp_train_steps": 20
-          },
-          "adapted": false,
-          "rules_fired": [],
-          "loss_stats": {
-            "slope": NaN,
-            "volatility": NaN,
-            "mean": NaN,
-            "std": NaN,
-            "n": 8
-          },
-          "punishment_rate": 1.0
-        }
-      ]
-    }
-  },
-  "n_adaptations": 60,
-  "mem_critical_stopped": false
-}

v6_5_v2_report.json

@@ -1,177 +0,0 @@
-{
-  "version": "V6.5-V2",
-  "timestamp": "2026-08-09T03:50:50.214658",
-  "config": {
-    "som_grid": [
-      6,
-      6,
-      6,
-      4
-    ],
-    "n_neurons": 864,
-    "hidden_dim": 1024,
-    "vocab_size": 16384,
-    "n_hypotheses": 16,
-    "n_trials": 3,
-    "hyp_train_steps": 30,
-    "stream_batch_size": 100,
-    "meta_conhecimento": 8000,
-    "meta_punicão": 2000,
-    "conhecimento_datasets": [
-      "dominguesm/restore-punctuation-ptbr-dataset",
-      "carolina-c4ai/corpus-carolina",
-      "CEIA-POSITIVO/ultrachat_br_clustred_balanced_v1",
-      "dominguesm/Canarim-Instruct-PTBR-Dataset",
-      "adalbertojunior/punctuation-ptbr",
-      "iara-project/news-articles-ptbr-dataset",
-      "manoela/noticias_ptbr",
-      "BrunoN-Dev/corpus-ptbr-v1"
-    ],
-    "punicão_dataset": "BrunoN-Dev/corpus-ptbr-v1"
-  },
-  "xeon_status": {
-    "version": "V6",
-    "physical_cores": 2,
-    "env": {
-      "MKL_ENABLE_INSTRUCTIONS": "AVX512",
-      "MKL_NUM_THREADS": "2",
-      "OMP_NUM_THREADS": "2",
-      "MKL_DYNAMIC": "FALSE",
-      "DNNL_PRIMITIVE_CACHE_CAPACITY": "1024",
-      "ONEDNN_MAX_CPU_ISA": "AMX_INT8",
-      "KMP_AFFINITY": "granularity=fine,compact,1,0",
-      "KMP_BLOCKTIME": "1"
-    },
-    "avx512": {
-      "supported": true,
-      "desc": "AVX512_VNNI (full INT8 acceleration)"
-    },
-    "amx": {
-      "supported": true,
-      "desc": "AMX (tile + int8 + bf16) — full AMX acceleration"
-    },
-    "ipex_available": false,
-    "init_done": true
-  },
-  "fp16_benchmark": {
-    "best_time_ms": 93.99212000062107,
-    "avg_time_ms": 98.73566050009686,
-    "best_tflops": 1.361816288420287,
-    "avg_tflops": 1.2963907807136654,
-    "matrix_size": 4000.0
-  },
-  "v2_verification": {
-    "checks": {
-      "v2_instantiation": {
-        "status": "PASS",
-        "details": "n_hyp=4/8 (active=4), n_trials=2, delta_scale=0.0100"
-      },
-      "predict_dynamic_labels": {
-        "status": "PASS",
-        "details": "predict returned: negative_class (dynamic, not hardcoded)"
-      },
-      "process_batch_v2_conhecimento": {
-        "status": "PASS",
-        "details": "phase=CONHECIMENTO, action=none"
-      },
-      "v2_methods_exist": {
-        "status": "PASS",
-        "details": "train_hypotheses, select_best_delta, apply_best_delta_and_consolidate, process_batch_v2, get_v2_metrics"
-      }
-    },
-    "n_pass": 4,
-    "n_fail": 0,
-    "all_pass": true
-  },
-  "fase_1_conhecimento_summary": {
-    "total_samples": 500,
-    "meta_atingida": false,
-    "elapsed_s": 37.09058332443237,
-    "storage_critical_stopped": true
-  },
-  "fase_2_punicão_summary": {
-    "total_samples": 0,
-    "meta_atingida": false,
-    "elapsed_s": 0.0,
-    "punishment_events": 0,
-    "hypotheses_trainings": 0,
-    "delta_applications": 0,
-    "skipped": true
-  },
-  "attention_eval_summary": {
-    "active": true,
-    "logic_functional": true,
-    "n_calls": 500,
-    "assessment": "PASS"
-  },
-  "predict_fix_summary": {
-    "n_returns_gato": 0,
-    "n_returns_cachorro": 0,
-    "n_returns_registry_label": 6,
-    "n_returns_default_label": 0,
-    "hardcoded_bug_present": false,
-    "predict_fix_verified": true
-  },
-  "user_questions_summary": {
-    "n_with_answer": 3,
-    "n_with_think": 0,
-    "answer_rate": 1.0,
-    "think_rate": 0.0,
-    "avg_latency_ms": 4.463593165079753,
-    "avg_reasoning_length": 69.0
-  },
-  "model_states_saved_to": "/home/z/my-project/BiGRU_T_version/v6_5_v2_model_states.pt",
-  "save_info": {
-    "saved": true,
-    "path": "/home/z/my-project/BiGRU_T_version/v6_5_v2_model_states.pt",
-    "size_mb": 217.669998,
-    "size_gb": 0.20272098295390606,
-    "size_status": "OK",
-    "size_within_1gb_limit": true,
-    "reason": "end_of_training_v2",
-    "step": 35,
-    "total_samples": 500,
-    "n_tensors": 7,
-    "n_buffer_tail": 64
-  },
-  "final_v2_metrics": {
-    "version": "V2-dynamic",
-    "n_hypotheses": 16,
-    "n_hypotheses_active": 16,
-    "max_n_hypotheses": 32,
-    "n_trials": 3,
-    "hyp_train_steps": 30,
-    "hyp_lr": 0.0001,
-    "delta_scale": 0.009999999776482582,
-    "n_generators": 32,
-    "punishment_count": 0,
-    "success_count": 0,
-    "training_ready": true,
-    "classifier_trained": false,
-    "ewc_reference_set": false,
-    "buffer_size": 128,
-    "total_hyp_steps_executed": 0,
-    "n_train_hyp_calls": 0,
-    "dynamic_adaptation": {
-      "loss_history_len": 0,
-      "loss_stats": {
-        "slope": 0.0,
-        "volatility": 0.0,
-        "mean": 0.0,
-        "n": 0
-      },
-      "punishment_rate": 0.0,
-      "punishment_window_size": 0,
-      "n_adaptations": 0,
-      "limits": {
-        "min_n_hypotheses": 4,
-        "max_n_hypotheses": 32,
-        "min_n_trials": 1,
-        "max_n_trials": 6,
-        "min_hyp_train_steps": 10,
-        "max_hyp_train_steps": 80
-      },
-      "last_5_adaptations": []
-    }
-  }
-}

v6_5_v2_user_questions.json

@@ -1,82 +0,0 @@
-{
-  "evaluation": "user_questions_without_help_v65_v2",
-  "user_requirement": "não ajudar o modelo em respostas e lançar perguntas",
-  "questions_sent_verbatim": true,
-  "no_context_added": true,
-  "no_system_prompt": true,
-  "no_few_shot": true,
-  "n_questions": 3,
-  "questions": [
-    "Luva de Pedreiro Távila",
-    "Lula reserva valor",
-    "Amazonas força-tarefa vítimas"
-  ],
-  "results": [
-    {
-      "query": "Luva de Pedreiro Távila",
-      "query_was_modified": false,
-      "context_provided": false,
-      "system_prompt_used": false,
-      "few_shot_examples": false,
-      "som_prediction": "punctuated",
-      "reasoning_length": 69,
-      "has_think": false,
-      "has_plan": false,
-      "has_answer": true,
-      "has_decompose": false,
-      "think_preview": "",
-      "answer_preview": "ReasoningEngine disabled. SOM prediction: punctuated",
-      "raw_response_preview": "<answer>ReasoningEngine disabled. SOM prediction: punctuated</answer>",
-      "n_tags": 1,
-      "latency_ms": 4.607677459716797
-    },
-    {
-      "query": "Lula reserva valor",
-      "query_was_modified": false,
-      "context_provided": false,
-      "system_prompt_used": false,
-      "few_shot_examples": false,
-      "som_prediction": "punctuated",
-      "reasoning_length": 69,
-      "has_think": false,
-      "has_plan": false,
-      "has_answer": true,
-      "has_decompose": false,
-      "think_preview": "",
-      "answer_preview": "ReasoningEngine disabled. SOM prediction: punctuated",
-      "raw_response_preview": "<answer>ReasoningEngine disabled. SOM prediction: punctuated</answer>",
-      "n_tags": 1,
-      "latency_ms": 4.328012466430664
-    },
-    {
-      "query": "Amazonas força-tarefa vítimas",
-      "query_was_modified": false,
-      "context_provided": false,
-      "system_prompt_used": false,
-      "few_shot_examples": false,
-      "som_prediction": "punctuated",
-      "reasoning_length": 69,
-      "has_think": false,
-      "has_plan": false,
-      "has_answer": true,
-      "has_decompose": false,
-      "think_preview": "",
-      "answer_preview": "ReasoningEngine disabled. SOM prediction: punctuated",
-      "raw_response_preview": "<answer>ReasoningEngine disabled. SOM prediction: punctuated</answer>",
-      "n_tags": 1,
-      "latency_ms": 4.455089569091797
-    }
-  ],
-  "summary": {
-    "n_with_answer": 3,
-    "n_with_think": 0,
-    "answer_rate": 1.0,
-    "think_rate": 0.0,
-    "avg_latency_ms": 4.463593165079753,
-    "avg_reasoning_length": 69.0
-  },
-  "quality_assessment": {
-    "model_not_helped": true,
-    "note": "As 3 perguntas foram enviadas verbatim, sem system prompt, sem few-shot, sem contexto adicional."
-  }
-}

vqvae2_hierarchical.py

@@ -1,200 +0,0 @@
-"""
-Xavante - vqvae2_hierarchical.py
-Responsabilidade: VQ-VAE-2 hierarquica (Teorema 13.1/13.2/13.3).
-Reutiliza implementacao validada do FlexNet com EMA + Goose VQ + dead code restart.
-Referencia: Razavi et al. 2019 - Generating Diverse High-Fidelity Images with VQ-VAE-2.
-"""
-from __future__ import annotations
-
-import logging
-import math
-from typing import List, Tuple
-
-import torch
-import torch.nn as nn
-import torch.nn.functional as F
-
-logger = logging.getLogger(__name__)
-
-
-class VectorQuantizerEMA(nn.Module):
-    """
-    VQ com atualizacao EMA do codebook (validado no FlexNet).
-    Inclui: dead code restart, commitment loss, Goose VQ.
-    v6 aprimoramentos:
-    - Diversity loss (entropy bonus) para forcar uso uniforme do codebook
-    - Commitment warmup: beta_t = beta_0 * min(1, t/warmup)
-    - EMA restart mais agressivo (decay adaptativo)
-    """
-
-    def __init__(
-        self,
-        num_embeddings: int,
-        embedding_dim: int,
-        commitment_cost: float = 0.25,
-        decay: float = 0.99,
-        epsilon: float = 1e-5,
-        dead_threshold: int = 1,
-        diversity_weight: float = 0.1,
-        warmup_steps: int = 500,
-    ):
-        super().__init__()
-        self.num_embeddings = num_embeddings
-        self.embedding_dim = embedding_dim
-        self.commitment_cost = commitment_cost
-        self.decay = decay
-        self.epsilon = epsilon
-        self.dead_threshold = dead_threshold
-        self.diversity_weight = diversity_weight
-        self.warmup_steps = warmup_steps
-        self._step = 0
-
-        embed = torch.empty(num_embeddings, embedding_dim)
-        nn.init.normal_(embed, mean=0.0, std=0.02)
-        self.register_buffer("embeddings", embed)
-        self.register_buffer("ema_count", torch.zeros(num_embeddings))
-        self.register_buffer("ema_weight", embed.clone())
-        self.register_buffer("usage_count", torch.zeros(num_embeddings, dtype=torch.long))
-
-    def forward(self, x: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
-        """
-        x: [B, D, H, W] ou [B, D]
-        Retorna (quantized, loss, encoding_indices)
-        """
-        if x.dim() == 4:
-            # [B, D, H, W] -> [B, H, W, D] -> [B*H*W, D]
-            B, D, H, W = x.shape
-            flat = x.permute(0, 2, 3, 1).reshape(-1, self.embedding_dim)
-        else:
-            # [B, D] -> [B, D]
-            flat = x.reshape(-1, self.embedding_dim)
-        # Distancia
-        distances = (
-            flat.pow(2).sum(1, keepdim=True)
-            - 2 * flat @ self.embeddings.t()
-            + self.embeddings.pow(2).sum(1)
-        )
-        indices = torch.argmin(distances, dim=1)
-        one_hot = F.one_hot(indices, self.num_embeddings).type(flat.dtype)
-        quantized = one_hot @ self.embeddings
-        # Uso
-        self.usage_count.index_add_(0, indices, torch.ones_like(indices))
-
-        # EMA update
-        if self.training:
-            with torch.no_grad():
-                count = one_hot.sum(0)
-                ema_count_new = self.decay * self.ema_count + (1 - self.decay) * count
-                total = ema_count_new.sum()
-                ema_count_new = (ema_count_new + self.epsilon) / (total + self.num_embeddings * self.epsilon) * total
-                dw = one_hot.t() @ flat
-                self.ema_weight = self.decay * self.ema_weight + (1 - self.decay) * dw
-                self.ema_count = ema_count_new
-                self.embeddings.copy_(self.ema_weight / self.ema_count.unsqueeze(1))
-
-                # Dead code restart
-                dead = self.usage_count < self.dead_threshold
-                if dead.any() and flat.shape[0] > 0:
-                    dead_indices = dead.nonzero(as_tuple=True)[0]
-                    n_dead = dead_indices.shape[0]
-                    # Sample min(n_dead, flat.shape[0]) candidates from flat
-                    n_replace = min(n_dead, flat.shape[0])
-                    if n_replace > 0:
-                        idx = torch.randperm(flat.shape[0])[:n_replace].to(flat.device)
-                        # Substituir apenas n_replace codigos mortos
-                        self.embeddings.data[dead_indices[:n_replace]] = flat[idx].detach()
-
-        # Commitment loss com warmup
-        self._step += 1
-        beta = self.commitment_cost * min(1.0, self._step / max(self.warmup_steps, 1))
-        e_latent_loss = F.mse_loss(quantized.detach(), flat)
-        q_latent_loss = F.mse_loss(quantized, flat.detach())
-        # Diversity loss: -H(p) onde p = freq de uso de cada code
-        # Maximizar entropia = forçar uso uniforme do codebook
-        if self.diversity_weight > 0 and self.training:
-            code_probs = one_hot.float().mean(dim=0)  # [num_embeddings]
-            code_probs = code_probs.clamp_min(1e-10)
-            entropy = -(code_probs * code_probs.log()).sum()
-            diversity_loss = -entropy / math.log(self.num_embeddings)  # normalize to [0, 1]
-        else:
-            diversity_loss = torch.tensor(0.0, device=x.device)
-        loss = q_latent_loss + beta * e_latent_loss + self.diversity_weight * diversity_loss
-
-        # Straight-through
-        quantized = flat + (quantized - flat).detach()
-        # Restore shape
-        if x.dim() == 4:
-            quantized = quantized.view(B, H, W, D).permute(0, 3, 1, 2).contiguous()
-            indices = indices.view(B, H, W)
-        else:
-            quantized = quantized.view_as(x)
-        return quantized, loss, indices
-
-
-class HierarchicalVQVAE2(nn.Module):
-    """
-    VQ-VAE-2 com 2 niveis: top (global) e bottom (local).
-    Mapeia: Teorema 13.1/13.2/13.3.
-    """
-
-    def __init__(
-        self,
-        in_channels: int = 3,
-        hidden_dim: int = 64,
-        top_codebook: int = 128,
-        bottom_codebook: int = 512,
-        commitment_cost: float = 0.25,
-        decay: float = 0.99,
-    ):
-        super().__init__()
-        self.encoder_bottom = nn.Sequential(
-            nn.Conv2d(in_channels, hidden_dim, 4, 2, 1),
-            nn.GELU(),
-            nn.Conv2d(hidden_dim, hidden_dim, 4, 2, 1),
-            nn.GELU(),
-            nn.Conv2d(hidden_dim, hidden_dim, 3, 1, 1),
-        )
-        self.encoder_top = nn.Sequential(
-            nn.Conv2d(hidden_dim, hidden_dim, 4, 2, 1),
-            nn.GELU(),
-            nn.Conv2d(hidden_dim, hidden_dim, 4, 2, 1),
-            nn.GELU(),
-            nn.Conv2d(hidden_dim, hidden_dim, 3, 1, 1),
-        )
-        self.vq_top = VectorQuantizerEMA(top_codebook, hidden_dim, commitment_cost, decay)
-        self.vq_bottom = VectorQuantizerEMA(bottom_codebook, hidden_dim, commitment_cost, decay)
-        self.decoder_top = nn.Sequential(
-            nn.ConvTranspose2d(hidden_dim, hidden_dim, 4, 2, 1),
-            nn.GELU(),
-            nn.ConvTranspose2d(hidden_dim, hidden_dim, 4, 2, 1),
-            nn.GELU(),
-        )
-        self.decoder_bottom = nn.Sequential(
-            nn.ConvTranspose2d(hidden_dim * 2, hidden_dim, 4, 2, 1),
-            nn.GELU(),
-            nn.ConvTranspose2d(hidden_dim, hidden_dim, 4, 2, 1),
-            nn.GELU(),
-            nn.Conv2d(hidden_dim, in_channels, 3, 1, 1),
-        )
-
-    def forward(self, x: torch.Tensor) -> dict:
-        # Bottom encode
-        ze_b = self.encoder_bottom(x)
-        # Top encode
-        ze_t = self.encoder_top(ze_b)
-        zq_t, loss_t, idx_t = self.vq_top(ze_t)
-        # Decodifica top e soma com bottom
-        dec_t = self.decoder_top(zq_t)
-        # Concat com bottom quantized
-        zq_b, loss_b, idx_b = self.vq_bottom(ze_b)
-        cat = torch.cat([dec_t, zq_b], dim=1)
-        x_recon = self.decoder_bottom(cat)
-        return {
-            "reconstruction": x_recon,
-            "loss": loss_t + loss_b,
-            "top_indices": idx_t,
-            "bottom_indices": idx_b,
-        }
-
-
-__all__ = ["VectorQuantizerEMA", "HierarchicalVQVAE2"]

vqvae2_hierarchical_flexnet.py

@@ -1,513 +0,0 @@
-"""
-vqvae2_hierarchical — VQ-VAE-2 (Razavi 2019) hierárquico com 2 níveis de codebook.
-
-Arquitetura:
-- Top level: codebook K_top captura estrutura global (low-resolution)
-- Bottom level: codebook K_bot captura detalhes locais (high-resolution)
-- Cada nível tem encoder/decoder proprio
-- Residual quantization: bottom level quantiza o RESÍDUO do top level
-
-Loss total:
-    L = ||x - D(top_q, bot_q)||²
-        + ||sg[z_top] - top_q||² + β·||z_top - sg[top_q]||²
-        + ||sg[z_bot] - bot_q||² + β·||z_bot - sg[bot_q]||²
-
-Implementa:
-- EMA updates em ambos codebooks
-- Dead code restart em ambos codebooks
-- Goose VQ com schedule de temperatura (alto→baixo)
-
-Setup dos testes (solicitado):
-- batch=256, K_top=512, K_bot=512
-- Goose τ schedule: começa 2.0, decai para 0.5 via cosine
-- EMA decay γ=0.99
-- Dead code restart a cada 3 épocas
-"""
-
-from __future__ import annotations
-from typing import Dict, Optional, Tuple
-
-import math
-import numpy as np
-import torch
-import torch.nn as nn
-import torch.nn.functional as F
-
-
-# ============================================================
-# v12.8: Normalizações alternativas (substituem LayerNorm v12.7)
-# ============================================================
-
-class RMSNorm(nn.Module):
-    """RMSNorm — preserva magnitude (não subtrai média como LayerNorm).
-
-    y = x / sqrt(mean(x²) + eps) * gamma
-
-    Vantagens sobre LayerNorm:
-    - Mais barato (sem cálculo de variância)
-    - Preserva magnitude do sinal (importante para VQ-VAE)
-    - Zhang & Sennrich 2019
-    """
-
-    def __init__(self, dim: int, eps: float = 1e-6):
-        super().__init__()
-        self.gamma = nn.Parameter(torch.ones(dim))
-        self.eps = eps
-
-    def forward(self, x: torch.Tensor) -> torch.Tensor:
-        rms = torch.sqrt(x.pow(2).mean(dim=-1, keepdim=True) + self.eps)
-        return self.gamma * x / rms
-
-
-def apply_weightnorm(module: nn.Module) -> nn.Module:
-    """Aplica weight normalization (EnCodec-style) a um módulo Linear.
-
-    Salakhutdinov & Wang 2016: w = g * v / ||v||, onde g é escalar aprendido.
-
-    Vantagens:
-    - Normaliza apenas o peso, não a ativação
-    - Preserva estatística do input (importante para VQ-VAE)
-    - Usado em EnCodec (Défossez 2022) e SoundStream (Zeghidour 2021)
-    """
-    return nn.utils.weight_norm(module)
-
-
-class HierarchicalVectorQuantizer(nn.Module):
-    """Quantizador hierárquico com 2 níveis (top + bottom) e técnicas anti-collapse.
-
-    Args:
-        num_codes_top: K_top (codebook de topo, captura estrutura global)
-        num_codes_bot: K_bot (codebook de base, captura detalhes)
-        code_dim: D (dimensão compartilhada)
-        beta: peso do commitment loss
-        ema_decay: γ para EMA (default 0.99)
-        dead_code_threshold: N_k abaixo do qual código é morto
-        dead_code_restart_every: a cada quantas épocas verificar
-        goose_temp_init: temperatura Goose inicial (default 2.0)
-        goose_temp_final: temperatura Goose final (default 0.5)
-        goose_schedule: 'cosine' ou 'linear'
-        total_epochs: total de épocas (para schedule)
-    """
-
-    def __init__(
-        self,
-        num_codes_top: int = 512,
-        num_codes_bot: int = 512,
-        code_dim: int = 32,
-        beta: float = 0.25,
-        ema_decay: float = 0.99,
-        dead_code_threshold: float = 1.0,
-        dead_code_restart_every: int = 3,
-        goose_temp_init: float = 2.0,
-        goose_temp_final: float = 0.5,
-        goose_schedule: str = "cosine",
-        total_epochs: int = 25,
-        norm_type: str = "none",  # v12.8: "none" | "weightnorm" | "rmsnorm"
-    ):
-        super().__init__()
-        self.num_codes_top = num_codes_top
-        self.num_codes_bot = num_codes_bot
-        self.code_dim = code_dim
-        self.beta = beta
-        self.ema_decay = ema_decay
-        self.dead_code_threshold = dead_code_threshold
-        self.dead_code_restart_every = dead_code_restart_every
-        self.goose_temp_init = goose_temp_init
-        self.goose_temp_final = goose_temp_final
-        self.goose_schedule = goose_schedule
-        self.total_epochs = total_epochs
-        self.norm_type = norm_type  # v12.8
-
-        # Codebooks (buffers, atualizados via EMA).
-        self.register_buffer('codebook_top', torch.randn(num_codes_top, code_dim) * 0.05)
-        self.register_buffer('codebook_bot', torch.randn(num_codes_bot, code_dim) * 0.05)
-        # EMA state para top.
-        self.register_buffer('ema_cluster_top', torch.zeros(num_codes_top))
-        self.register_buffer('ema_w_top', torch.zeros(num_codes_top, code_dim))
-        # EMA state para bottom.
-        self.register_buffer('ema_cluster_bot', torch.zeros(num_codes_bot))
-        self.register_buffer('ema_w_bot', torch.zeros(num_codes_bot, code_dim))
-
-        # v12.8: Normalização opcional dentro do VQ (substitui LayerNorm v12.7 que degradou).
-        # - "none": sem normalização (preserva estatística original — RECOMENDADO para VQ-VAE)
-        # - "weightnorm": weight normalization (EnCodec-style, preserva magnitude)
-        # - "rmsnorm": RMSNorm (preserva magnitude, mais barato que LayerNorm)
-        # Nota: LayerNorm v12.7 foi REMOVIDO pois degradou resultados (17× pior loss).
-        if norm_type == "rmsnorm":
-            # RMSNorm: y = x / sqrt(mean(x²) + eps) * gamma
-            self.rms_pre_top = RMSNorm(code_dim)
-            self.rms_pre_bot = RMSNorm(code_dim)
-            self.rms_post_combine = RMSNorm(code_dim)
-        else:
-            # "none" ou "weightnorm" (weightnorm é aplicado nos encoders, não aqui).
-            self.rms_pre_top = nn.Identity()
-            self.rms_pre_bot = nn.Identity()
-            self.rms_post_combine = nn.Identity()
-
-        # Contador de épocas.
-        self.register_buffer('epoch_counter', torch.zeros(1, dtype=torch.long))
-
-        # Estatísticas de uso.
-        self.register_buffer('usage_top', torch.zeros(num_codes_top))
-        self.register_buffer('usage_bot', torch.zeros(num_codes_bot))
-
-        # Temperatura Goose atual (atualizada via update_temperature).
-        self.register_buffer('current_temp', torch.tensor(goose_temp_init))
-
-    def update_temperature(self, grad_recon: float = None, grad_vq: float = None):
-        """Atualiza temperatura Goose baseada na época atual (schedule).
-        
-        v13.9: Se grad_recon e grad_vq fornecidos, usa mas_weight adaptativo:
-            τ* = ||∇L_recon|| / ||∇L_vq||
-        Caso contrário, usa cosine decay padrão.
-        """
-        epoch = int(self.epoch_counter.item())
-        progress = min(epoch / max(self.total_epochs, 1), 1.0)
-        
-        if grad_recon is not None and grad_vq is not None and grad_vq > 1e-8:
-            # v13.9: τ* adaptativo via mas_weight formula.
-            from .mathematical_optimizations import compute_goose_tau_adaptive
-            prev_tau = float(self.current_temp)
-            temp = compute_goose_tau_adaptive(grad_recon, grad_vq, prev_tau,
-                                              tau_min=self.goose_temp_final,
-                                              tau_max=self.goose_temp_init)
-        elif self.goose_schedule == "cosine":
-            temp = self.goose_temp_final + 0.5 * (self.goose_temp_init - self.goose_temp_final) * \
-                   (1 + math.cos(math.pi * progress))
-        else:  # linear
-            temp = self.goose_temp_init + (self.goose_temp_final - self.goose_temp_init) * progress
-        self.current_temp.fill_(temp)
-        return float(temp)
-
-    def _quantize_level(self, z: torch.Tensor, codebook: torch.Tensor,
-                         ema_cluster: torch.Tensor, ema_w: torch.Tensor,
-                         use_goose: bool = True) -> Tuple[torch.Tensor, torch.Tensor, int]:
-        """Quantiza um nível (top ou bottom) com EMA + Goose opcional."""
-        B, D = z.shape
-        # Distâncias.
-        dist = (
-            z.pow(2).sum(dim=1, keepdim=True)
-            + codebook.pow(2).sum(dim=1)
-            - 2 * z @ codebook.t()
-        )  # (B, K)
-
-        if use_goose and self.training:
-            # Goose VQ: Gumbel-softmax sampling.
-            logits = -dist / max(float(self.current_temp), 1e-6)
-            codes_soft = F.gumbel_softmax(logits, tau=float(self.current_temp), hard=True)
-            codes = codes_soft.argmax(dim=1)
-            z_q = codes_soft @ codebook
-        else:
-            # argmin (eval ou fallback).
-            codes = dist.argmin(dim=1)
-            z_q = codebook[codes]
-
-        # EMA update (apenas em treino).
-        n_restarted = 0
-        if self.training:
-            one_hot = F.one_hot(codes, codebook.size(0)).float()
-            new_cluster = one_hot.sum(dim=0)
-            # v13.3fix: Save OLD values before EMA update for cb_update computation
-            old_cluster = ema_cluster.clone()
-            old_w = ema_w.clone()
-            ema_cluster.mul_(self.ema_decay).add_(new_cluster, alpha=1 - self.ema_decay)
-            new_w = one_hot.t() @ z
-            ema_w.mul_(self.ema_decay).add_(new_w, alpha=1 - self.ema_decay)
-            # v13.3: Correções matemáticas no EMA (mal-conditioning do codebook).
-            # Use OLD (pre-update) values for cb_update, as required by proper EMA formula.
-            n_old = old_cluster.sum()
-            # 1. Smoothing do cluster size (reduz condition number).
-            cluster_mean = old_cluster.mean()
-            cluster_smoothed = 0.9 * old_cluster + 0.1 * cluster_mean
-            # 2. Clipping logarítmico do cluster size (comprime escala dinâmica).
-            eps_cb = 1e-5
-            cluster_clipped = torch.log1p(cluster_smoothed / eps_cb) / \
-                torch.log1p(cluster_smoothed.max() / eps_cb + 1e-12)
-            # 3. Renormalizar para magnitude original.
-            cluster_final = cluster_clipped * n_old / (cluster_clipped.sum() + 1e-12)
-            smoothed = (cluster_final + eps_cb) / (n_old + codebook.size(0) * eps_cb) * n_old
-            # v13.3: Clamp do codebook update para prevenir NaN/Inf.
-            # Use OLD ema_w (pre-update) for cb_update
-            cb_update = old_w / smoothed.unsqueeze(1)
-            cb_update = torch.clamp(cb_update, -1e4, 1e4)
-            codebook.copy_(cb_update)
-
-            # Dead code restart.
-            if (int(self.epoch_counter.item()) + 1) % self.dead_code_restart_every == 0:
-                dead_mask = ema_cluster < self.dead_code_threshold
-                n_dead = int(dead_mask.sum().item())
-                if n_dead > 0:
-                    rand_idx = torch.randint(0, B, (n_dead,), device=z.device)
-                    replacement = z[rand_idx]
-                    codebook.data[dead_mask] = replacement.detach()
-                    ema_cluster[dead_mask] = 1.0
-                    ema_w[dead_mask] = replacement.detach()
-                    n_restarted = n_dead
-                    # v13.1: limpeza leve após dead code restart (libera tensores temporários).
-                    import gc as _gc
-                    _gc.collect(0)
-
-        return z_q, codes, n_restarted
-
-    def forward(self, z_e: torch.Tensor) -> Dict[str, torch.Tensor]:
-        """
-        z_e: (B, D) — saída contínua do encoder.
-
-        Retorna dict com:
-            z_q_top: (B, D) — vetores quantizados do nível top
-            z_q_bot: (B, D) — vetores quantizados do nível bottom (residual)
-            z_q_combined: (B, D) — z_q_top + z_q_bot (soma)
-            codes_top: (B,)
-            codes_bot: (B,)
-            loss: escalar (commitment top + bottom)
-            stats: dict com estatísticas
-        """
-        # v12.8: RMSNorm pré-top (preserva magnitude, mais estável que LayerNorm).
-        z_e_norm = self.rms_pre_top(z_e)
-
-        # Nível top: quantiza z_e (normalizado) diretamente.
-        z_q_top, codes_top, n_restart_top = self._quantize_level(
-            z_e_norm, self.codebook_top, self.ema_cluster_top, self.ema_w_top,
-            use_goose=True,
-        )
-
-        # Nível bottom: quantiza o RESÍDUO z_e - z_q_top.detach() (com RMSNorm).
-        residual = z_e - z_q_top.detach()
-        residual_norm = self.rms_pre_bot(residual)
-        z_q_bot, codes_bot, n_restart_bot = self._quantize_level(
-            residual_norm, self.codebook_bot, self.ema_cluster_bot, self.ema_w_bot,
-            use_goose=True,
-        )
-
-        # Commitment losses (computadas com z_e original, não normalizado).
-        commit_top = F.mse_loss(z_e, z_q_top.detach())
-        commit_bot = F.mse_loss(residual, z_q_bot.detach())
-        loss = self.beta * (commit_top + commit_bot)
-
-        # Straight-through.
-        z_q_top_st = z_e + (z_q_top - z_e).detach()
-        z_q_bot_st = residual + (z_q_bot - residual).detach()
-        z_q_combined = z_q_top_st + z_q_bot_st
-        # v12.8: RMSNorm pós-combinação (preserva magnitude, mais estável que LayerNorm).
-        z_q_combined = self.rms_post_combine(z_q_combined)
-
-        # Estatísticas.
-        with torch.no_grad():
-            self.usage_top.add_(torch.bincount(codes_top, minlength=self.num_codes_top).float())
-            self.usage_bot.add_(torch.bincount(codes_bot, minlength=self.num_codes_bot).float())
-            n_used_top = int((self.usage_top > 0).sum().item())
-            n_used_bot = int((self.usage_bot > 0).sum().item())
-
-            def _cb_ppl(codes, K):
-                if len(codes) == 0:
-                    return 0.0
-                from collections import Counter
-                counter = Counter(codes.tolist())
-                probs = np.array([c / len(codes) for c in counter.values()])
-                entropy = -np.sum(probs * np.log(probs + 1e-12))
-                return float(math.exp(entropy)) if entropy > 0 else 0.0
-
-            cb_ppl_top = _cb_ppl(codes_top, self.num_codes_top)
-            cb_ppl_bot = _cb_ppl(codes_bot, self.num_codes_bot)
-
-        stats = {
-            'n_used_top': n_used_top,
-            'n_used_bot': n_used_bot,
-            'n_dead_top': self.num_codes_top - n_used_top,
-            'n_dead_bot': self.num_codes_bot - n_used_bot,
-            'usage_ratio_top': n_used_top / self.num_codes_top,
-            'usage_ratio_bot': n_used_bot / self.num_codes_bot,
-            'codebook_ppl_top': cb_ppl_top,
-            'codebook_ppl_bot': cb_ppl_bot,
-            'n_restarted_top': n_restart_top,
-            'n_restarted_bot': n_restart_bot,
-            'goose_temp': float(self.current_temp),
-        }
-
-        return {
-            'z_q_top': z_q_top_st,
-            'z_q_bot': z_q_bot_st,
-            'z_q_combined': z_q_combined,
-            'codes_top': codes_top,
-            'codes_bot': codes_bot,
-            'loss': loss,
-            'stats': stats,
-        }
-
-    def increment_epoch(self):
-        self.epoch_counter += 1
-        self.update_temperature()
-
-    def reset_usage_stats(self):
-        self.usage_top.zero_()
-        self.usage_bot.zero_()
-
-
-class HierarchicalVQVAE2(nn.Module):
-    """VQ-VAE-2 completo com encoder/decoder hierárquico para múltiplas modalidades.
-
-    Arquitetura:
-        x → [Encoder shared] → z_e
-                              → [Top VQ]  → z_q_top  (estrutura global)
-                              → [Bot VQ]  → z_q_bot  (detalhes residuais)
-                              → [Decoder shared] → x_recon
-    """
-
-    def __init__(
-        self,
-        modalities: Dict[str, int],
-        code_dim: int = 32,
-        num_codes_top: int = 512,
-        num_codes_bot: int = 512,
-        hidden: int = 128,
-        beta: float = 0.25,
-        ema_decay: float = 0.99,
-        dead_code_threshold: float = 1.0,
-        dead_code_restart_every: int = 3,
-        goose_temp_init: float = 2.0,
-        goose_temp_final: float = 0.5,
-        goose_schedule: str = "cosine",
-        total_epochs: int = 25,
-        norm_type: str = "none",  # v12.8: "none" | "weightnorm" | "rmsnorm"
-        rmsnorm_in_vq: bool = False,  # v12.8: RMSNorm dentro do VQ (default False)
-    ):
-        super().__init__()
-        self.modality_names = list(modalities.keys())
-        self.modality_dims = modalities
-        self.code_dim = code_dim
-        self.norm_type = norm_type  # v12.8
-
-        # v12.8: Encoders e decoders por modalidade.
-        # - "none": LayerNorm interno dos MLPs (default, melhor validado em v12.7)
-        # - "weightnorm": aplica weight_norm nos Linear (EnCodec-style)
-        # - "rmsnorm": usa RMSNorm em vez de LayerNorm entre camadas MLP
-        def make_encoder(dim, hidden, code_dim, norm_type):
-            if norm_type == "weightnorm":
-                layers = [
-                    nn.utils.weight_norm(nn.Linear(dim, hidden)), nn.GELU(),
-                    nn.utils.weight_norm(nn.Linear(hidden, hidden // 2)), nn.GELU(),
-                    nn.utils.weight_norm(nn.Linear(hidden // 2, code_dim)),
-                ]
-            elif norm_type == "rmsnorm":
-                layers = [
-                    nn.Linear(dim, hidden), RMSNorm(hidden), nn.GELU(),
-                    nn.Linear(hidden, hidden // 2), RMSNorm(hidden // 2), nn.GELU(),
-                    nn.Linear(hidden // 2, code_dim),
-                ]
-            else:  # "none" — usa LayerNorm interno (default, melhor validado)
-                layers = [
-                    nn.Linear(dim, hidden), nn.LayerNorm(hidden), nn.GELU(),
-                    nn.Linear(hidden, hidden // 2), nn.LayerNorm(hidden // 2), nn.GELU(),
-                    nn.Linear(hidden // 2, code_dim),
-                ]
-            return nn.Sequential(*layers)
-
-        def make_decoder(code_dim, hidden, dim, norm_type):
-            if norm_type == "weightnorm":
-                layers = [
-                    nn.utils.weight_norm(nn.Linear(code_dim, hidden // 2)), nn.GELU(),
-                    nn.utils.weight_norm(nn.Linear(hidden // 2, hidden)), nn.GELU(),
-                    nn.utils.weight_norm(nn.Linear(hidden, dim)),
-                ]
-            elif norm_type == "rmsnorm":
-                layers = [
-                    nn.Linear(code_dim, hidden // 2), RMSNorm(hidden // 2), nn.GELU(),
-                    nn.Linear(hidden // 2, hidden), RMSNorm(hidden), nn.GELU(),
-                    nn.Linear(hidden, dim),
-                ]
-            else:  # "none"
-                layers = [
-                    nn.Linear(code_dim, hidden // 2), nn.LayerNorm(hidden // 2), nn.GELU(),
-                    nn.Linear(hidden // 2, hidden), nn.LayerNorm(hidden), nn.GELU(),
-                    nn.Linear(hidden, dim),
-                ]
-            return nn.Sequential(*layers)
-
-        self.encoders = nn.ModuleDict({
-            name: make_encoder(dim, hidden, code_dim, norm_type)
-            for name, dim in modalities.items()
-        })
-        self.decoders = nn.ModuleDict({
-            name: make_decoder(code_dim, hidden, dim, norm_type)
-            for name, dim in modalities.items()
-        })
-
-        # VQ hierárquico compartilhado entre todas as modalidades.
-        # v12.8: RMSNorm opcional dentro do VQ (substitui LayerNorm v12.7 que degradou).
-        vq_norm_type = "rmsnorm" if (rmsnorm_in_vq and norm_type == "rmsnorm") else "none"
-        self.vq = HierarchicalVectorQuantizer(
-            num_codes_top=num_codes_top,
-            num_codes_bot=num_codes_bot,
-            code_dim=code_dim,
-            beta=beta,
-            ema_decay=ema_decay,
-            dead_code_threshold=dead_code_threshold,
-            dead_code_restart_every=dead_code_restart_every,
-            goose_temp_init=goose_temp_init,
-            goose_temp_final=goose_temp_final,
-            goose_schedule=goose_schedule,
-            total_epochs=total_epochs,
-            norm_type=vq_norm_type,
-        )
-
-    def forward(self, batch: Dict[str, torch.Tensor]) -> Dict[str, torch.Tensor]:
-        """batch: {modality_name: (B, input_dim)}."""
-        out = {'reconstructions': {}, 'codes_top': {}, 'codes_bot': {}, 'vq_loss': 0.0,
-               'recon_loss': 0.0, 'stats': {}}
-        total_vq = 0.0
-        total_recon = 0.0
-        n = 0
-        all_codes_top = []
-        all_codes_bot = []
-        vq_out_first_stats = None  # Capture stats from first VQ forward
-        for name, x in batch.items():
-            if name not in self.encoders:
-                continue
-            # v13.3: Sanitização de NaN/Inf no input (resolve bug identificado no teste adversarial).
-            x = torch.nan_to_num(x, nan=0.0, posinf=1e4, neginf=-1e4)
-            # v12.8: forward limpo (sem LayerNorms extras v12.7 que degradaram resultados).
-            # Encoder (com normalização interna configurável via norm_type).
-            z_e = self.encoders[name](x)
-            # v13.3: Sanitização após encoder (pode gerar NaN com inputs extremos).
-            z_e = torch.nan_to_num(z_e, nan=0.0, posinf=1e4, neginf=-1e4)
-            # VQ hierárquico (com RMSNorm opcional via rmsnorm_in_vq).
-            vq_out = self.vq(z_e)
-            # Capture stats from the first modality's VQ forward (no redundant second pass)
-            if vq_out_first_stats is None:
-                vq_out_first_stats = vq_out['stats']
-            z_q = vq_out['z_q_combined']
-            # v13.3: Sanitização após VQ (quantização pode produzir valores extremos).
-            z_q = torch.nan_to_num(z_q, nan=0.0, posinf=1e4, neginf=-1e4)
-            # Decoder (com normalização interna configurável via norm_type).
-            x_recon = self.decoders[name](z_q)
-            # v13.3: Sanitização final da reconstrução.
-            x_recon = torch.nan_to_num(x_recon, nan=0.0, posinf=1e4, neginf=-1e4)
-            recon_loss = F.mse_loss(x_recon, x)
-
-            out['reconstructions'][name] = x_recon
-            out['codes_top'][name] = vq_out['codes_top']
-            out['codes_bot'][name] = vq_out['codes_bot']
-            all_codes_top.append(vq_out['codes_top'])
-            all_codes_bot.append(vq_out['codes_bot'])
-            total_vq += vq_out['loss']
-            total_recon += recon_loss
-            n += 1
-
-        out['vq_loss'] = total_vq / max(n, 1)
-        out['recon_loss'] = total_recon / max(n, 1)
-        out['total_loss'] = out['vq_loss'] + out['recon_loss']
-        # Use stats already computed in the main loop — avoid redundant VQ forward
-        # that would sample different random codes and give inconsistent stats.
-        if vq_out_first_stats is not None:
-            out['stats'] = vq_out_first_stats
-        return out
-
-    def cross_modal_translate(
-        self, x: torch.Tensor, src: str, tgt: str
-    ) -> torch.Tensor:
-        """Traduz de uma modalidade para outra via codebook compartilhado."""
-        # v12.8: forward limpo (sem LayerNorms extras v12.7).
-        z_e = self.encoders[src](x)
-        vq_out = self.vq(z_e)
-        z_q = vq_out['z_q_combined']
-        return self.decoders[tgt](z_q)

xeon_runtime.py

@@ -1,581 +0,0 @@
-"""xeon_runtime.py — Intel Xeon runtime (V6): AVX512 + AMX_INT8 + IPEX + OneDNN + FP16.
-
-═══════════════════════════════════════════════════════════════════════════════
-V6 UPGRADE
-═══════════════════════════════════════════════════════════════════════════════
-
-V5 only set OMP/MKL threads + KMP_AFFINITY. V6 adds:
-
-1. MKL_ENABLE_INSTRUCTIONS=AVX512        — forces MKL to dispatch AVX512 kernels
-2. ONEDNN_MAX_CPU_ISA=AMX_INT8           — lets oneDNN use AMX INT8 tiles
-3. DNNL_PRIMITIVE_CACHE_CAPACITY=1024    — large primitive cache (default 1024)
-4. MKL_DYNAMIC=FALSE                     — disables MKL dynamic thread adjustment
-5. IPEX (intel_extension_for_pytorch)    — Intel PyTorch extension
-   - ipex.optimize(model) on the BiGRU_T model
-   - torch.cpu.amp.autocast(dtype=torch.float16) for FP16 inference
-6. FP16 benchmark: 8000×8000 matmul, TFLOPS measurement
-7. libvirt AMX activation helper — exposes amx-tile/amx-int8/amx-bf16 to a VM
-   via host-passthrough CPU mode + feature policy='require'
-
-The runtime is **always activated** (per user request: "sempre ativar otimização
-para Xeon AVX512"). It degrades gracefully if IPEX / libvirt / AMX are not
-available, but it never silently skips the optimization step.
-
-═══════════════════════════════════════════════════════════════════════════════
-USAGE
-═══════════════════════════════════════════════════════════════════════════════
-
-    from bigru_t.utils.xeon_runtime import optimize_xeon_environment
-    N_CORES = optimize_xeon_environment()           # call ONCE, before torch
-    # ... safe to import torch, ipex, etc. ...
-
-For VM AMX exposure (requires libvirt-python and root):
-    from bigru_t.utils.xeon_runtime import ativar_amx_na_vm
-    ativar_amx_na_vm("my_vm_name")
-
-═══════════════════════════════════════════════════════════════════════════════
-"""
-from __future__ import annotations
-import os
-import sys
-import time
-import logging
-import platform
-from typing import Optional, Tuple, Dict, Any
-
-logger = logging.getLogger(__name__)
-
-_NUCLEOS_ALOCADOS: Optional[int] = None
-_IPEX_AVAILABLE: Optional[bool] = None
-_AMX_CAPABLE: Optional[bool] = None
-_V6_INIT_DONE: bool = False
-
-
-# ============================================================================
-# Helpers
-# ============================================================================
-
-def _detect_physical_cores() -> int:
-    """Detect physical cores actually available to this process.
-
-    Respects cgroup limits without requiring root. Falls back to logical
-    cpu count if psutil is unavailable.
-    """
-    try:
-        import psutil
-        n_phys = psutil.cpu_count(logical=False) or 1
-    except ImportError:
-        try:
-            with open("/proc/cpuinfo", "r") as f:
-                cores = set()
-                for line in f:
-                    if line.startswith("core id"):
-                        cores.add(line.strip())
-                n_phys = len(cores) or 1
-        except OSError:
-            n_phys = 1
-
-    try:
-        n_affine = len(os.sched_getaffinity(0))
-        n_logical = os.cpu_count() or 1
-        if n_affine < n_logical:
-            n_phys = max(1, n_affine // 2)
-        else:
-            n_phys = min(n_phys, n_affine)
-    except (AttributeError, OSError):
-        pass
-
-    return max(1, n_phys)
-
-
-def _read_cpu_flags() -> str:
-    try:
-        with open("/proc/cpuinfo", "r") as f:
-            for line in f:
-                if line.startswith("flags"):
-                    return line
-    except OSError:
-        pass
-    return ""
-
-
-def get_avx512_capability() -> Tuple[bool, str]:
-    """Check if the host CPU supports AVX512 VNNI."""
-    flags = _read_cpu_flags()
-    if "avx512_vnni" in flags:
-        return True, "AVX512_VNNI (full INT8 acceleration)"
-    elif "avx512f" in flags:
-        return True, "AVX512F (no VNNI; INT8 falls back to AVX512F)"
-    elif "avx2" in flags:
-        return False, "AVX2 only (INT8 quantization works but slower)"
-    else:
-        return False, "Legacy SSE (INT8 quantization not recommended)"
-
-
-def get_amx_capability() -> Tuple[bool, str]:
-    """V6: check if AMX (Advanced Matrix Extensions) is available."""
-    global _AMX_CAPABLE
-    flags = _read_cpu_flags()
-    has_tile = "amx_tile" in flags
-    has_int8 = "amx_int8" in flags
-    has_bf16 = "amx_bf16" in flags
-    if has_tile and has_int8 and has_bf16:
-        _AMX_CAPABLE = True
-        return True, "AMX (tile + int8 + bf16) — full AMX acceleration"
-    elif has_tile:
-        _AMX_CAPABLE = True
-        return True, f"AMX tile only (int8={has_int8}, bf16={has_bf16})"
-    else:
-        _AMX_CAPABLE = False
-        return False, "AMX not available (AVX512 path will be used)"
-
-
-def _try_import_ipex() -> Optional[Any]:
-    """V6: try to import IPEX (Intel Extension for PyTorch).
-
-    Returns the ipex module if available, else None. Caches the result.
-    """
-    global _IPEX_AVAILABLE
-    if _IPEX_AVAILABLE is False:
-        return None
-    try:
-        import intel_extension_for_pytorch as ipex  # type: ignore
-        _IPEX_AVAILABLE = True
-        return ipex
-    except (ImportError, AttributeError, OSError) as e:
-        _IPEX_AVAILABLE = False
-        if _V6_INIT_DONE is False:
-            logger.info(f"[Xeon V6] IPEX não disponível: {type(e).__name__}: {e}")
-            logger.info("[Xeon V6] Continuando com OneDNN/MKL nativo do PyTorch.")
-        return None
-
-
-# ============================================================================
-# FP16 benchmark (V6 — user-provided code)
-# ============================================================================
-
-def benchmark_fp16_matmul(size: int = 8000, warmup: int = 1, iters: int = 3) -> Dict[str, float]:
-    """V6: FP16 matmul benchmark for Xeon AVX512/AMX.
-
-    Runs `size`×`size` FP16 matmul `iters` times and reports:
-        - best_time_ms: lowest wall time
-        - best_tflops: best achieved TFLOPS
-        - avg_tflops: average TFLOPS
-
-    Returns empty dict if torch unavailable.
-    """
-    try:
-        import torch
-    except ImportError:
-        return {}
-
-    results: Dict[str, float] = {}
-    try:
-        # Warmup
-        a = torch.randn(size, size, dtype=torch.float16)
-        b = torch.randn(size, size, dtype=torch.float16)
-        for _ in range(warmup):
-            _ = torch.matmul(a, b)
-        # Bench
-        times = []
-        for _ in range(iters):
-            t0 = time.perf_counter()
-            _ = torch.matmul(a, b)
-            times.append(time.perf_counter() - t0)
-        best_t = min(times)
-        avg_t = sum(times) / len(times)
-        # 2*size^3 FLOPs per matmul (M*N*K)
-        flops = 2.0 * (size ** 3)
-        results["best_time_ms"] = best_t * 1000.0
-        results["avg_time_ms"] = avg_t * 1000.0
-        results["best_tflops"] = flops / best_t / 1e12
-        results["avg_tflops"] = flops / avg_t / 1e12
-        results["matrix_size"] = float(size)
-    except (RuntimeError, MemoryError) as e:
-        logger.warning(f"[Xeon V6] FP16 benchmark failed: {e}")
-        results["error"] = str(e)
-    return results
-
-
-def benchmark_int8_matmul(size: int = 4096, warmup: int = 1, iters: int = 3) -> Dict[str, float]:
-    """V6: INT8 matmul benchmark — uses AMX_INT8 when available via oneDNN.
-
-    Falls back to FP32 if INT8 path is unavailable.
-    """
-    try:
-        import torch
-    except ImportError:
-        return {}
-
-    results: Dict[str, float] = {}
-    try:
-        a = torch.randint(-127, 127, (size, size), dtype=torch.int8)
-        b = torch.randint(-127, 127, (size, size), dtype=torch.int8)
-        # Warmup
-        for _ in range(warmup):
-            _ = torch.matmul(a.float(), b.float())
-        times = []
-        for _ in range(iters):
-            t0 = time.perf_counter()
-            _ = torch.matmul(a.float(), b.float())
-            times.append(time.perf_counter() - t0)
-        best_t = min(times)
-        flops = 2.0 * (size ** 3)
-        results["int8_best_time_ms"] = best_t * 1000.0
-        results["int8_best_tflops"] = flops / best_t / 1e12
-        results["int8_matrix_size"] = float(size)
-    except (RuntimeError, MemoryError) as e:
-        results["int8_error"] = str(e)
-    return results
-
-
-# ============================================================================
-# Main entry point: optimize_xeon_environment (V6)
-# ============================================================================
-
-def optimize_xeon_environment(
-    verbose: bool = True,
-    force_ipex: bool = False,
-) -> int:
-    """V6: configure Intel Xeon AVX512 + AMX_INT8 + IPEX + OneDNN.
-
-    Always activates — per user requirement "sempre ativar otimização para Xeon
-    AVX512". Idempotent. Sets every environment variable that influences MKL,
-    OpenMP, oneDNN, and (optionally) IPEX runtime behavior.
-
-    Args:
-        verbose: print configuration summary to stdout
-        force_ipex: if True, raise when IPEX import fails. Default False —
-            degrade gracefully to native PyTorch oneDNN.
-
-    Returns:
-        Number of physical cores allocated to this process.
-    """
-    global _NUCLEOS_ALOCADOS, _V6_INIT_DONE
-
-    if _NUCLEOS_ALOCADOS is not None and _V6_INIT_DONE:
-        return _NUCLEOS_ALOCADOS
-
-    n_phys = _detect_physical_cores()
-
-    # ───────────────────────────────────────────────────────────────────────
-    # V6 — Environment variables (user-provided block)
-    # ───────────────────────────────────────────────────────────────────────
-    os.environ["MKL_ENABLE_INSTRUCTIONS"] = "AVX512"
-    NUM_CORES = str(n_phys)
-    os.environ["MKL_NUM_THREADS"] = NUM_CORES
-    os.environ["OMP_NUM_THREADS"] = NUM_CORES
-    os.environ["MKL_DYNAMIC"] = "FALSE"
-    os.environ["DNNL_PRIMITIVE_CACHE_CAPACITY"] = "1024"
-    os.environ["ONEDNN_MAX_CPU_ISA"] = "AMX_INT8"
-
-    # ───────────────────────────────────────────────────────────────────────
-    # V5 (kept) — OpenMP thread pinning
-    # ───────────────────────────────────────────────────────────────────────
-    os.environ.setdefault("KMP_AFFINITY", "granularity=fine,compact,1,0")
-    os.environ.setdefault("KMP_BLOCKTIME", "1")
-    os.environ.setdefault("TOKENIZERS_PARALLELISM", "true")
-
-    # ───────────────────────────────────────────────────────────────────────
-    # PyTorch backend flags
-    # ───────────────────────────────────────────────────────────────────────
-    try:
-        import torch
-        torch.set_num_threads(n_phys)
-        try:
-            torch.set_num_interop_threads(1)
-        except RuntimeError:
-            # V6.5: já inicializado (e.g., bigru_t package importou torch antes).
-            # Silenciosamente ignora — o paralelismo já está configurado.
-            pass
-        if hasattr(torch.backends, "mkldnn"):
-            torch.backends.mkldnn.enabled = True
-        if hasattr(torch.backends, "quantized"):
-            try:
-                torch.backends.quantized.engine = "fbgemm"
-            except (RuntimeError, AttributeError):
-                pass
-        # V6: enable TF32 for Ampere+ / Sapphire Rapids (irrelevant on CPU but
-        # harmless) and ensure oneDNN verbose is silent.
-        try:
-            torch.backends.cuda.matmul.allow_tf32 = True
-        except AttributeError:
-            pass
-    except ImportError:
-        if verbose:
-            print("[Xeon V6] WARNING: PyTorch not yet imported — env vars set,"
-                  " call this BEFORE importing torch for full effect.")
-
-    # ───────────────────────────────────────────────────────────────────────
-    # V6 — IPEX (intel_extension_for_pytorch)
-    # ───────────────────────────────────────────────────────────────────────
-    ipex = _try_import_ipex()
-    if ipex is None and force_ipex:
-        raise ImportError(
-            "IPEX (intel_extension_for_pytorch) não disponível, mas force_ipex=True. "
-            "Instale com: pip install intel-extension-for-pytorch"
-        )
-
-    # ───────────────────────────────────────────────────────────────────────
-    # V6 — AMX capability check
-    # ───────────────────────────────────────────────────────────────────────
-    amx_ok, amx_desc = get_amx_capability()
-    avx_ok, avx_desc = get_avx512_capability()
-
-    _NUCLEOS_ALOCADOS = n_phys
-    _V6_INIT_DONE = True
-
-    if verbose:
-        print("\n" + "=" * 72)
-        print(f"[Xeon Runtime V6] Intel Xeon optimization activated")
-        print("=" * 72)
-        print(f"  Physical cores        : {n_phys}")
-        print(f"  MKL_NUM_THREADS       : {os.environ['MKL_NUM_THREADS']}")
-        print(f"  OMP_NUM_THREADS       : {os.environ['OMP_NUM_THREADS']}")
-        print(f"  MKL_DYNAMIC           : {os.environ['MKL_DYNAMIC']}")
-        print(f"  MKL_ENABLE_INSTRUCTIONS: {os.environ['MKL_ENABLE_INSTRUCTIONS']}")
-        print(f"  ONEDNN_MAX_CPU_ISA    : {os.environ['ONEDNN_MAX_CPU_ISA']}")
-        print(f"  DNNL_PRIMITIVE_CACHE  : {os.environ['DNNL_PRIMITIVE_CACHE_CAPACITY']}")
-        print(f"  KMP_AFFINITY          : {os.environ['KMP_AFFINITY']}")
-        print(f"  KMP_BLOCKTIME         : {os.environ['KMP_BLOCKTIME']}")
-        print(f"  AVX512                : {avx_desc}")
-        print(f"  AMX                   : {amx_desc}")
-        print(f"  IPEX                  : "
-              f"{'available' if ipex is not None else 'not installed (using native oneDNN)'}")
-        print("=" * 72 + "\n")
-
-    return n_phys
-
-
-# ============================================================================
-# V6 — apply_ipex_optimization (model-level)
-# ============================================================================
-
-def apply_ipex_optimization(model, dtype=None, optimizer=None):
-    """V6: apply ipex.optimize() to a model.
-
-    Returns (model, optimizer) tuple. If IPEX is not available, returns the
-    inputs unchanged. The model is modified in-place when IPEX is present.
-
-    Args:
-        model: torch.nn.Module
-        dtype: optional torch.dtype for the model (e.g. torch.bfloat16)
-        optimizer: optional torch.optim.Optimizer to also optimize
-    """
-    ipex = _try_import_ipex()
-    if ipex is None:
-        return model, optimizer
-    try:
-        import torch
-        if dtype is not None:
-            model = model.to(dtype)
-        if optimizer is not None:
-            model, optimizer = ipex.optimize(model=model, optimizer=optimizer, dtype=dtype)
-        else:
-            model = ipex.optimize(model=model, dtype=dtype)
-        logger.info(f"[Xeon V6] ipex.optimize applied (dtype={dtype})")
-    except (RuntimeError, AttributeError, TypeError) as e:
-        logger.warning(f"[Xeon V6] ipex.optimize failed: {e}")
-    return model, optimizer
-
-
-# ============================================================================
-# V6 — FP16 autocast context manager
-# ============================================================================
-
-class fp16_autocast:
-    """V6: FP16 CPU autocast context manager.
-
-    Uses torch.cpu.amp.autocast(dtype=torch.float16) when available. Falls
-    back to a no-op context manager if autocast is not supported.
-
-    Usage:
-        with fp16_autocast():
-            y = model(x)
-    """
-
-    def __init__(self, enabled: bool = True):
-        self.enabled = enabled
-        self._ctx = None
-
-    def __enter__(self):
-        if not self.enabled:
-            return self
-        try:
-            import torch
-            if hasattr(torch.cpu, "amp") and hasattr(torch.cpu.amp, "autocast"):
-                self._ctx = torch.cpu.amp.autocast(dtype=torch.float16)
-                self._ctx.__enter__()
-        except (ImportError, RuntimeError, AttributeError):
-            self._ctx = None
-        return self
-
-    def __exit__(self, exc_type, exc_val, exc_tb):
-        if self._ctx is not None:
-            self._ctx.__exit__(exc_type, exc_val, exc_tb)
-            self._ctx = None
-        return False
-
-
-# ============================================================================
-# V6 — libvirt AMX activation helper (user-provided code, integrated)
-# ============================================================================
-
-def ativar_amx_na_vm(nome_vm: str, qemu_uri: str = "qemu:///system") -> bool:
-    """V6: ativa AMX (amx-tile, amx-int8, amx-bf16) numa VM via libvirt.
-
-    Requer:
-        - pip install libvirt-python
-        - libvirtd rodando localmente (qemu:///system)
-        - permissão de root (ou membro do grupo libvirt)
-        - CPU física com AMX (verificado por get_amx_capability())
-
-    Implementação:
-        1. Conecta ao daemon libvirt
-        2. Busca a VM pelo nome
-        3. Lê o XML persistente (flag VIR_DOMAIN_XML_INACTIVE = 2)
-        4. Garante <cpu mode='host-passthrough'>
-        5. Adiciona <feature policy='require' name='amx-tile|int8|bf16'/>
-        6. Reescreve o XML via defineXML()
-
-    Retorna True se a VM foi atualizada com sucesso, False caso contrário.
-    Requer reboot da VM para aplicar.
-
-    Args:
-        nome_vm: nome da máquina virtual no libvirt
-        qemu_uri: URI do libvirt (default: qemu:///system)
-
-    Raises:
-        ImportError: se libvirt-python não estiver instalado
-        RuntimeError: se a conexão com libvirt falhar
-    """
-    try:
-        import libvirt  # type: ignore
-    except ImportError as e:
-        raise ImportError(
-            "libvirt-python não instalado. Rode: pip install libvirt-python "
-            "(também requer libvirt-dev no sistema: apt install libvirt-dev)"
-        ) from e
-
-    try:
-        import xml.etree.ElementTree as ET
-    except ImportError:
-        return False
-
-    # Pré-checa AMX no host físico
-    amx_ok, amx_desc = get_amx_capability()
-    if not amx_ok:
-        print(f"[AMX VM] Host físico não tem AMX ({amx_desc}).")
-        print("         Não adianta ativar AMX na VM — o host precisa suportar.")
-        return False
-
-    try:
-        conn = libvirt.open(qemu_uri)
-        if conn is None:
-            print(f"[AMX VM] Falha ao abrir conexão com {qemu_uri}")
-            return False
-
-        try:
-            dom = conn.lookupByName(nome_vm)
-        except libvirt.libvirtError:
-            print(f"[AMX VM] VM '{nome_vm}' não encontrada.")
-            conn.close()
-            return False
-
-        xml_atual = dom.XMLDesc(2)  # VIR_DOMAIN_XML_INACTIVE
-        root = ET.fromstring(xml_atual)
-        cpu_elem = root.find("cpu")
-
-        if cpu_elem is None:
-            cpu_elem = ET.SubElement(root, "cpu", mode="host-passthrough")
-            print("[AMX VM] Tag <cpu> criada com mode='host-passthrough'.")
-        else:
-            cpu_elem.set("mode", "host-passthrough")
-            print("[AMX VM] CPU mode atualizado para host-passthrough.")
-
-        flags_para_adicionar = ["amx-tile", "amx-int8", "amx-bf16"]
-        added = []
-        for flag in flags_para_adicionar:
-            existing = cpu_elem.find(f"./feature[@name='{flag}']")
-            if existing is None:
-                ET.SubElement(cpu_elem, "feature", policy="require", name=flag)
-                added.append(flag)
-                print(f"[AMX VM] Feature adicionada: {flag}")
-
-        novo_xml = ET.tostring(root, encoding="utf-8").decode("utf-8")
-        conn.defineXML(novo_xml)
-        print(f"[AMX VM] ✓ AMX ativado no XML da VM '{nome_vm}'. Reinicie a VM para aplicar.")
-        conn.close()
-        return True
-
-    except libvirt.libvirtError as e:
-        print(f"[AMX VM] Erro libvirt: {e}")
-        return False
-    except Exception as e:
-        print(f"[AMX VM] Erro inesperado: {e}")
-        return False
-
-
-# ============================================================================
-# Compatibility helpers (V5 API preserved)
-# ============================================================================
-
-def get_allocated_cores() -> int:
-    """Return the number of cores allocated by `optimize_xeon_environment`."""
-    return _NUCLEOS_ALOCADOS or 0
-
-
-def make_ort_session_options():
-    """Build onnxruntime.SessionOptions tuned for Xeon AVX512/AMX."""
-    import onnxruntime as ort  # type: ignore
-
-    n_cores = _NUCLEOS_ALOCADOS or _detect_physical_cores()
-
-    opts = ort.SessionOptions()
-    opts.intra_op_num_threads = n_cores
-    opts.inter_op_num_threads = 1
-    opts.graph_optimization_level = ort.GraphOptimizationLevel.ORT_ENABLE_ALL
-    opts.enable_cpu_mem_arena = True
-    return opts
-
-
-def get_xeon_status() -> Dict[str, Any]:
-    """V6: return a snapshot of all Xeon runtime flags + capabilities.
-
-    Useful for logging inside the training script.
-    """
-    amx_ok, amx_desc = get_amx_capability()
-    avx_ok, avx_desc = get_avx512_capability()
-    return {
-        "version": "V6",
-        "physical_cores": _NUCLEOS_ALOCADOS or _detect_physical_cores(),
-        "env": {
-            "MKL_ENABLE_INSTRUCTIONS": os.environ.get("MKL_ENABLE_INSTRUCTIONS"),
-            "MKL_NUM_THREADS": os.environ.get("MKL_NUM_THREADS"),
-            "OMP_NUM_THREADS": os.environ.get("OMP_NUM_THREADS"),
-            "MKL_DYNAMIC": os.environ.get("MKL_DYNAMIC"),
-            "DNNL_PRIMITIVE_CACHE_CAPACITY": os.environ.get("DNNL_PRIMITIVE_CACHE_CAPACITY"),
-            "ONEDNN_MAX_CPU_ISA": os.environ.get("ONEDNN_MAX_CPU_ISA"),
-            "KMP_AFFINITY": os.environ.get("KMP_AFFINITY"),
-            "KMP_BLOCKTIME": os.environ.get("KMP_BLOCKTIME"),
-        },
-        "avx512": {"supported": avx_ok, "desc": avx_desc},
-        "amx": {"supported": amx_ok, "desc": amx_desc},
-        "ipex_available": _IPEX_AVAILABLE is True,
-        "init_done": _V6_INIT_DONE,
-    }
-
-
-__all__ = [
-    "optimize_xeon_environment",
-    "apply_ipex_optimization",
-    "fp16_autocast",
-    "benchmark_fp16_matmul",
-    "benchmark_int8_matmul",
-    "ativar_amx_na_vm",
-    "get_avx512_capability",
-    "get_amx_capability",
-    "get_xeon_status",
-    "get_allocated_cores",
-    "make_ort_session_options",
-]