V6.5-V2-dynamic: upload batch (scripts + state + model) [12 files]

hyp_t.py

@@ -0,0 +1,100 @@
+"""hyp_t.py — HypT: cabeça de hipótese (correção delta). Implementa o Lema 3.
+
+A Hipótese é treinada para cancelar o ruído de quantização W8A8:
+    y_final = y_hat + delta
+onde delta = hyp_T(o).
+
+Diferenças vs. TrainT:
+- Recebe `o` opcionalmente destacado (stop_grad) para isolar o sinal de
+  correção do gradiente principal
+- Mesma arquitetura (TransformerEncoder) — apenas o tratamento de gradiente
+  difere
+
+Reaproveita a filosofia do HypothesisNetwork do v13.9.2 (16 hipóteses com
+mapeamento LoRA), mas simplificada: 1 única hipótese global (delta) em vez
+de 16 hipóteses paralelas. A multiplexação é feita pelo OrqCell + Lema 1.
+
+============================================================================
+V6.4 — pgvector_lookup REMOVIDO (NÃO É MAIS NECESSÁRIO)
+============================================================================
+Conforme directive do usuário:
+  "pgvector_lookup não é mais necessário pela lógica do script seguinte"
+
+O KohonenLearningSystem (kohonen_refactored/kohonen_learning_system.py)
+contém o método find_bmu que realiza busca nearest-neighbor sobre o grid
+4D do SOM (864 neurônios para grid (6,6,6,4)), substituindo qualquer
+lookup pgvector externo. A decisão de aplicar punição é delegada ao
+protocolo interno do KohonenLearningSystem (1ª punição → activate_hypothesis,
+2ª punição → set_ewc_reference + reset).
+
+Consequentemente, HypT volta ao seu papel original: gerar a correção
+delta = hyp_T(o) sem consultar mapa externo. A integração com o SOM é
+feita no nível do trainer (KohonenLearningSystem.process_batch), não no
+nível da hipótese.
+
+Os contadores de punição (n_complete_skip, n_incomplete_apply, etc.)
+foram removidos pois não há mais decisão de skip baseada em pgvector.
+"""
+from __future__ import annotations
+
+import torch
+import torch.nn as nn
+
+
+class HypT(nn.Module):
+    """Transformer de hipótese: gera correção delta.
+
+    Args:
+        d_input: dimensão de entrada (saída do OrqCell = d_cache)
+        d_model: dimensão interna
+        nhead: nº de cabeças
+        d_ff: dimensão do FFN
+        output_dim: dimensão de saída (igual ao TrainT)
+        num_layers: nº de camadas
+        dropout: probabilidade de dropout
+
+    Forward:
+        o: (batch, d_input), stop_grad: bool → delta: (batch, output_dim)
+
+    Se stop_grad=True, aplica o.detach() no início — Lema 3 (isolar correção
+    do gradiente principal).
+    """
+
+    def __init__(
+        self,
+        d_input: int,
+        d_model: int,
+        nhead: int,
+        d_ff: int,
+        output_dim: int,
+        num_layers: int = 2,
+        dropout: float = 0.1,
+    ):
+        super().__init__()
+        self.d_input = d_input
+        self.d_model = d_model
+        self.output_dim = output_dim
+
+        self.proj_in = nn.Linear(d_input, d_model)
+        encoder_layer = nn.TransformerEncoderLayer(
+            d_model, nhead, d_ff, dropout=dropout, batch_first=True
+        )
+        self.encoder = nn.TransformerEncoder(encoder_layer, num_layers)
+        self.proj_out = nn.Linear(d_model, output_dim)
+
+    def forward(self, o: torch.Tensor, stop_grad: bool = True) -> torch.Tensor:
+        """o: (batch, d_input) → delta: (batch, output_dim)
+
+        V6.4: SEM consulta pgvector — delta é computado diretamente.
+        A decisão de aplicar ou não a correção é delegada ao
+        KohonenLearningSystem (que usa find_bmu internamente).
+        """
+        # Lema 3: stop_grad_hyp isola o sinal de correção
+        if stop_grad:
+            o = o.detach()
+
+        x = o.unsqueeze(1)               # (batch, 1, d_input)
+        x = self.proj_in(x)              # (batch, 1, d_model)
+        x = self.encoder(x)              # (batch, 1, d_model)
+        delta = self.proj_out(x.squeeze(1))  # (batch, output_dim)
+        return delta

streaming_datasets.py

@@ -0,0 +1,995 @@
+"""streaming_datasets_v13_9.py — Streaming dataset loader for v13.9.1 training.
+
+Carrega os 9 datasets especificados pelo usuário via streaming (IterableDataset),
+um por vez, exaustivamente dentro do limite max_samples_per_dataset.
+
+Datasets:
+  1. CEIA-POSITIVO/ultrachat_br_clustred_balanced_v1
+  2. Madras1/corpus-ptbr-v2
+  3. rhaymison/multmodal_175k_portuguese
+  4. TucanoBR/GigaVerbo
+  5. nvidia/OpenMathReasoning
+  6. MathLLMs/MathVision
+  7. nvidia/OpenMathInstruct-2
+  8. dominguesm/restore-punctuation-ptbr-dataset
+  9. carolina-c4ai/corpus-carolina
+
+Memory-efficient: streaming=True, no full materialization.
+Returns ProcessedSample with raw_text only (tokenizer handles encoding).
+"""
+from __future__ import annotations
+
+import logging
+import os
+import time
+import traceback
+from dataclasses import dataclass, field
+from typing import Any, Dict, Iterator, List, Optional, Tuple
+
+import torch
+
+logger = logging.getLogger(__name__)
+
+# Datasets V13.9.1 — exatamente os 9 especificados pelo usuário
+# V6.5-final: adicionado dominguesm/Canarim-Instruct-PTBR-Dataset
+# V6.5-7ds: adicionados adalbertojunior/punctuation-ptbr,
+#           iara-project/news-articles-ptbr-dataset, manoela/noticias_ptbr
+# V6.5-V2: adicionado BrunoN-Dev/corpus-ptbr-v1 como 8º dataset de CONHECIMENTO
+#          e usado como dataset de TREINAMENTO COM PUNIÇÃO
+DEFAULT_DATASETS = [
+    "CEIA-POSITIVO/ultrachat_br_clustred_balanced_v1",
+    "Madras1/corpus-ptbr-v2",
+    "rhaymison/multmodal_175k_portuguese",
+    "TucanoBR/GigaVerbo",
+    "nvidia/OpenMathReasoning",
+    "MathLLMs/MathVision",
+    "nvidia/OpenMathInstruct-2",
+    "dominguesm/restore-punctuation-ptbr-dataset",
+    "carolina-c4ai/corpus-carolina",
+    "dominguesm/Canarim-Instruct-PTBR-Dataset",
+    "adalbertojunior/punctuation-ptbr",
+    "iara-project/news-articles-ptbr-dataset",
+    "manoela/noticias_ptbr",
+    "BrunoN-Dev/corpus-ptbr-v1",
+]
+
+# V6.5-V2 — Sequência de 8 datasets para a fase CONHECIMENTO (na ordem exata
+# especificada pelo usuário). Meta mínima: 8000 samples ou mais.
+CONHECIMENTO_DATASETS_V2 = [
+    "dominguesm/restore-punctuation-ptbr-dataset",
+    "carolina-c4ai/corpus-carolina",
+    "CEIA-POSITIVO/ultrachat_br_clustred_balanced_v1",
+    "dominguesm/Canarim-Instruct-PTBR-Dataset",
+    "adalbertojunior/punctuation-ptbr",
+    "iara-project/news-articles-ptbr-dataset",
+    "manoela/noticias_ptbr",
+    "BrunoN-Dev/corpus-ptbr-v1",
+]
+
+# V6.5-V2 — Dataset para TREINAMENTO COM PUNIÇÃO (500 samples).
+# O usuário especificou: "dataset para TREINAMENTO COM PUNIÇÃO ATIVA
+# 'BrunoN-Dev/corpus-ptbr-v1'".
+PUNICAO_DATASET_V2 = "BrunoN-Dev/corpus-ptbr-v1"
+
+# Dataset format descriptors — Lista priorizada de campos de texto
+DATASET_FORMATS: Dict[str, Dict[str, Any]] = {
+    "CEIA-POSITIVO/ultrachat_br_clustred_balanced_v1": {
+        "type": "chat",
+        "text_fields": ["conversa", "text", "content", "conversation", "messages"],
+        "label_fields": [],
+        "split": "train",
+        "config": None,
+    },
+    "Madras1/corpus-ptbr-v2": {
+        "type": "text",
+        "text_fields": ["text", "content", "document", "body"],
+        "label_fields": [],
+        "split": "train",
+        "config": None,
+    },
+    "rhaymison/multmodal_175k_portuguese": {
+        "type": "multimodal",
+        "text_fields": ["description", "text", "question", "prompt", "instruction", "input"],
+        "image_fields": ["image", "image_url"],
+        "label_fields": ["answer", "response", "output"],
+        "split": "train",
+        "config": None,
+    },
+    "TucanoBR/GigaVerbo": {
+        "type": "text",
+        "text_fields": ["text", "content", "document", "body"],
+        "label_fields": [],
+        "split": "train",
+        "config": None,
+    },
+    "nvidia/OpenMathReasoning": {
+        "type": "math",
+        "text_fields": ["problem", "question", "input", "prompt"],
+        "label_fields": ["solution", "answer", "output", "response"],
+        # BUG FIX: OpenMathReasoning has splits ['cot', 'tir', 'genselect', 'additional_problems']
+        "split": "cot",  # Use 'cot' (chain-of-thought) as primary split
+        "config": None,
+    },
+    "MathLLMs/MathVision": {
+        "type": "math_multimodal",
+        "text_fields": ["question", "problem", "text", "query"],
+        "image_fields": ["image", "image_url"],
+        "label_fields": ["answer", "solution", "response"],
+        # BUG FIX: MathVision has only 'test' and 'testmini' splits (no train)
+        "split": "test",
+        "config": None,
+    },
+    "nvidia/OpenMathInstruct-2": {
+        "type": "math",
+        "text_fields": ["problem", "question", "input", "prompt"],
+        # V3 FIX: campos reais do dataset são generated_solution e expected_answer
+        "label_fields": ["generated_solution", "expected_answer", "solution", "answer", "output", "response"],
+        "split": "train",
+        "config": None,
+    },
+    "dominguesm/restore-punctuation-ptbr-dataset": {
+        "type": "punctuation",
+        "text_fields": ["text", "original", "unpunctuated", "input"],
+        "label_fields": ["punctuated", "restored", "target", "output"],
+        "split": "train",
+        "config": None,
+    },
+    "carolina-c4ai/corpus-carolina": {
+        "type": "text",
+        "text_fields": ["text", "content", "document", "body", "xml"],
+        "label_fields": [],
+        "split": "train",
+        "config": None,
+        # V13.9.2-carolina: dataset original usa script Python (não suportado em
+        # datasets 5.0+). Workaround: stream_xml_gz direto via lxml.iterparse,
+        # sem usar o script Python. Implementado em _stream_carolina_direct().
+    },
+    # ── V13.9.2-finetune-v2: novos datasets PT-BR ───────────────────────────
+    # Adicionados para fine-tuning contínuo do modelo v13.9.2 (preserva
+    # arquitetura e parâmetros). Nenhuma mudança estrutural — apenas novos
+    # descritores de formato para que stream_dataset() reconheça os datasets.
+    #
+    # V13.9.2-finetune-v3: padronização para o formato unificado
+    # "### Instruction:\n...\n\n### Response:\n..." conforme requisitado pelo
+    # usuário. Apenas os datasets com estrutura instruction/response (orion,
+    # cnmoro) usam o template; strak2005/bratao continuam como texto puro
+    # (corpus sem estrutura de instrução). Nenhuma mudança nos 9 datasets
+    # base — retrocompatibilidade total.
+    "orion-research/translations-en_US-pt_BR": {
+        "type": "translation",
+        # string = EN (instruction), string_translation = PT-BR (response).
+        # V13.9.2-finetune-v3: separa instruction/response para o template
+        # unificado em vez de concatenar como texto único.
+        "text_fields": ["string"],
+        "label_fields": ["string_translation"],
+        "split": "train",
+        "config": None,
+        "format_template": "instruction_response",
+        "instruction_prefix": "Translate the following text to Portuguese:",
+    },
+    "cnmoro/Instruct-PTBR-10M": {
+        "type": "instruct",
+        # NB: campos em MAIÚSCULAS no schema do dataset
+        # V13.9.2-finetune-v3: INSTRUCTION -> instruction, RESPONSE -> response
+        # via template unificado.
+        "text_fields": ["INSTRUCTION"],
+        "label_fields": ["RESPONSE"],
+        "split": "train",
+        "config": None,
+        "format_template": "instruction_response",
+        # V13.9.2-finetune-v2: dataset é um único parquet de 10 GB — streaming
+        # via datasets.load_dataset() é extremamente lento para materializar
+        # a primeira amostra. Fallback via HF datasets-server rows API (HTTP),
+        # que retorna batches de 100 rows rapidamente. Implementado em
+        # _stream_via_rows_api().
+        "loader": "rows_api",
+    },
+    "strak2005/corpus-ptbr-v1": {
+        "type": "text",
+        "text_fields": ["text", "content"],
+        "label_fields": [],
+        "split": "train",
+        "config": None,
+        # Fallback: se strak2005 cair, tenta bratao (mirror idêntico)
+        "fallback": "bratao/corpus-ptbr-v1",
+        # V13.9.2-finetune-v2: dataset tem 18 shards parquet de ~1GB cada.
+        # Streaming via datasets.load_dataset() não materializa primeira
+        # amostra em tempo aceitável. Fallback: baixa 1 shard localmente e
+        # itera com pyarrow. Implementado em _stream_via_parquet_first_shard().
+        "loader": "parquet_first_shard",
+    },
+    "bratao/corpus-ptbr-v1": {
+        "type": "text",
+        "text_fields": ["text", "content"],
+        "label_fields": [],
+        "split": "train",
+        "config": None,
+        "loader": "parquet_first_shard",
+    },
+    # ── V6.5: novos datasets para esgotar ────────────────────────────────
+    # User requirement: "esgotar 'dominguesm/restore-punctuation-ptbr-dataset'
+    # e 'carolina-c4ai/corpus-carolina' e 'nvidia/OpenMathInstruct-2' e
+    # 'nvidia/OpenMathReasoning' e 'CEIA-POSITIVO/ultrachat_br_clustred_balanced_v1'
+    # e 'Dexavator/English-PTBR'"
+    #
+    # Dexavator/English-PTBR: dataset de tradução EN->PT-BR.
+    # Estrutura típica: {"english": "...", "portuguese": "..."} ou
+    # {"en": "...", "pt": "..."}.
+    "Dexavator/English-PTBR": {
+        "type": "translation",
+        "text_fields": ["english", "en", "text", "source"],
+        "label_fields": ["portuguese", "pt", "target", "translation"],
+        "split": "train",
+        "config": None,
+        "format_template": "instruction_response",
+        "instruction_prefix": "Translate the following text to Portuguese:",
+    },
+    # ── V6.5-final: Canarim-Instruct-PTBR-Dataset ────────────────────────
+    # User requirement: "esgotar 'dominguesm/Canarim-Instruct-PTBR-Dataset'"
+    # Canarim-Instruct: dataset PT-BR de instruções (~430k samples).
+    # Estrutura típica: {"instruction": "...", "input": "...", "output": "..."}
+    # ou {"text": "...", "conversation": [{"role": "user", "content": "..."}, ...]}
+    "dominguesm/Canarim-Instruct-PTBR-Dataset": {
+        "type": "instruct",
+        "text_fields": ["instruction", "input", "text", "prompt", "question"],
+        "label_fields": ["output", "response", "answer"],
+        "split": "train",
+        "config": None,
+        "format_template": "instruction_response",
+        "instruction_prefix": "Instrução:",
+    },
+    # ── V6.5-7ds: novos 3 datasets PT-BR ─────────────────────────────────
+    # User requirement (latest): "streaming até esgotar nesta sequência
+    # 'dominguesm/restore-punctuation-ptbr-dataset' e
+    # 'carolina-c4ai/corpus-carolina' e
+    # 'CEIA-POSITIVO/ultrachat_br_clustred_balanced_v1' e
+    # 'dominguesm/Canarim-Instruct-PTBR-Dataset' e
+    # 'adalbertojunior/punctuation-ptbr' e
+    # 'iara-project/news-articles-ptbr-dataset' e
+    # 'manoela/noticias_ptbr'"
+    #
+    # adalbertojunior/punctuation-ptbr:
+    #   Dataset de restauração de pontuação PT-BR. Schema:
+    #     {id: str, tokens: list[str], pos_tags, chunk_tags, ner_tags}
+    #   O campo "tokens" é uma lista de palavras (sem pontuação). Precisamos
+    #   fazer join com espaços para obter texto bruto. Como o dataset usa
+    #   script Python (datasets 5.0+ não suporta), usamos rows_api loader.
+    #   Splits: train/validation/test com config "punctuation-ptbr".
+    "adalbertojunior/punctuation-ptbr": {
+        "type": "punctuation_tokens",
+        "text_fields": ["tokens"],  # list[str] — join com espaços
+        "label_fields": [],
+        "split": "train",
+        "config": "punctuation-ptbr",  # necessário para rows_api
+        "loader": "rows_api",
+        "join_tokens": True,  # flag especial para _normalize_sample
+    },
+    # iara-project/news-articles-ptbr-dataset:
+    #   Dataset de notícias PT-BR da Folha de S.Paulo (via IARA project).
+    #   Schema: {title, text, date, category, category_natural_language, link}
+    #   Streaming normal via parquet — primeira amostra materializa rápido.
+    #   "text" é o corpo da notícia, "title" é o título. Concatenamos ambos
+    #   via format_template="title_text_concat" para enriquecer contexto.
+    #   O campo "category" é usado como label (categoria da notícia).
+    "iara-project/news-articles-ptbr-dataset": {
+        "type": "news",
+        # text_fields em ordem de prioridade: "text" (corpo) primeiro
+        "text_fields": ["text", "title", "content", "body"],
+        "label_fields": ["category", "category_natural_language", "title"],
+        "split": "train",
+        "config": None,
+        # V6.5-7ds: title_text_concat usa title como label (prefixo)
+        # e text como corpo. Aqui label_fields pega "category" primeiro
+        # para dar contexto temático.
+        "format_template": "title_text_concat",
+    },
+    # manoela/noticias_ptbr:
+    #   Mirror do iara-project/news-articles-ptbr-dataset (mesmo parquet,
+    #   mesmo schema). Streaming normal via parquet.
+    "manoela/noticias_ptbr": {
+        "type": "news",
+        "text_fields": ["text", "title", "content", "body"],
+        "label_fields": ["category", "category_natural_language", "title"],
+        "split": "train",
+        "config": None,
+        "format_template": "title_text_concat",
+    },
+    # ── V6.5-V2: BrunoN-Dev/corpus-ptbr-v1 ───────────────────────────────
+    # User requirement: "dataset para TREINAMENTO COM PUNIÇÃO ATIVA
+    # 'BrunoN-Dev/corpus-ptbr-v1'" e o 8º dataset de CONHECIMENTO.
+    #
+    # BrunoN-Dev/corpus-ptbr-v1: corpus PT-BR genérico com 18 shards parquet
+    # (~1GB cada). Schema: {"text": "...", "meta": "..."}.
+    # Streaming via datasets.load_dataset(streaming=True) é lento para
+    # materializar a primeira amostra — usamos o loader parquet_first_shard
+    # (baixa apenas o primeiro shard e itera localmente com pyarrow).
+    # Sem label explícita — usamos hash do texto (paridade do nº de palavras)
+    # para gerar label binário (0/1) — não é dado sintético, é derivação
+    # algorítmica do conteúdo real do dataset.
+    "BrunoN-Dev/corpus-ptbr-v1": {
+        "type": "text",
+        "text_fields": ["text", "content", "body", "document"],
+        "label_fields": [],
+        "split": "train",
+        "config": None,
+        "loader": "parquet_first_shard",
+        # V6.5-V2: gera label binário determinístico via hash do texto
+        # (paridade do número de palavras) — não é dado sintético, é
+        # derivação algorítmica do conteúdo real do dataset.
+        "label_strategy": "hash_parity",
+    },
+}
+
+
+# Lista de arquivos XML.gz do Carolina que tentaremos baixar (ordem: menores
+# taxonomias primeiro para minimizar banda e RAM). Selecionamos 2 arquivos
+# por taxonomia para garantir diversidade sem explodir a memória.
+CAROLINA_XML_FILES = [
+    # taxonomia "datasets_and_other_corpora" (DAT = pt-BR notícias)
+    "corpus/datasets_and_other_corpora/pt-BR/DATa.xml.gz",
+    "corpus/datasets_and_other_corpora/pt-BR/DATb.xml.gz",
+    # taxonomia "wik" (wikis)
+    "corpus/wik/WIKa.xml.gz",
+    "corpus/wik/WIKb.xml.gz",
+    # taxonomia "uni" (university_domains)
+    "corpus/uni/UNIa.xml.gz",
+    "corpus/uni/UNIb.xml.gz",
+    # taxonomia "pub" (public_domain_works)
+    "corpus/pub/PUBa.xml.gz",
+    "corpus/pub/PUBb.xml.gz",
+]
+
+
+def _stream_carolina_direct(
+    max_samples: int,
+    hf_token: Optional[str] = None,
+    timeout_per_file: int = 60,
+) -> Iterator[Dict[str, Any]]:
+    """Workaround V13.9.2-carolina: carrega Carolina sem usar o script Python.
+
+    Baixa arquivos XML.gz diretamente do repo e faz parse streaming com
+    lxml.etree.iterparse (compatível com datasets 5.0+). Cada <TEI> vira um
+    documento de texto (concatenação dos <p> dentro de <body>).
+
+    Args:
+        max_samples: número máximo de amostras a retornar
+        hf_token: token HF opcional (dataset é público, mas token evita rate limit)
+        timeout_per_file: tempo máximo (segundos) por arquivo XML
+
+    Yields:
+        dict com chave "text" (compatível com _normalize_sample)
+    """
+    from huggingface_hub import hf_hub_download
+    import gzip
+    from lxml import etree
+
+    TEI_NS = "{http://www.tei-c.org/ns/1.0}"
+    BODY_P_TAG = f".//{TEI_NS}body/{TEI_NS}p"
+
+    count = 0
+    for xml_path in CAROLINA_XML_FILES:
+        if count >= max_samples:
+            return
+        try:
+            local_path = hf_hub_download(
+                repo_id="carolina-c4ai/corpus-carolina",
+                filename=xml_path,
+                repo_type="dataset",
+                token=hf_token,
+            )
+        except Exception as e:
+            logger.warning(f"Carolina: não foi possível baixar {xml_path}: {e}")
+            continue
+
+        try:
+            with gzip.open(local_path, "rb") as gz:
+                # iterparse streaming: não carrega árvore inteira na memória
+                for _, tei in etree.iterparse(
+                    gz, huge_tree=True, encoding="utf-8", tag=f"{TEI_NS}TEI"
+                ):
+                    if count >= max_samples:
+                        tei.clear()
+                        return
+                    # Extrai texto dos <p> dentro de <body>
+                    parts = []
+                    for p in tei.findall(BODY_P_TAG):
+                        if p.text:
+                            parts.append(p.text)
+                    text = " ".join(parts).strip()
+                    if text and len(text) >= 10:
+                        yield {"text": text, "meta": ""}
+                    tei.clear()  # libera memória da árvore TEI
+                    count += 1
+        except Exception as e:
+            logger.warning(f"Carolina: erro processando {xml_path}: {e}")
+            continue
+
+    logger.info(f"Carolina: streamou {count} amostras no total")
+
+
+# ═══════════════════════════════════════════════════════════════════════════
+# V13.9.2-finetune-v2: Loaders alternativos para datasets parquet grandes
+# onde datasets.load_dataset(streaming=True) é inviável (primeira amostra
+# demora minutos para materializar).
+# ═══════════════════════════════════════════════════════════════════════════
+
+def _stream_via_rows_api(
+    dataset_name: str,
+    max_samples: int,
+    hf_token: Optional[str] = None,
+    batch_size: int = 100,
+    max_offset: int = 100_000,
+    config: Optional[str] = None,
+    split: str = "train",
+) -> Iterator[Dict[str, Any]]:
+    """Carrega amostras via HF datasets-server /rows API (HTTP).
+
+    Usado para datasets cujo parquet é muito grande para streaming eficiente
+    (ex: cnmoro/Instruct-PTBR-10M com 10 GB em arquivo único) OU datasets
+    que usam script Python como loader (não suportado em datasets 5.0+,
+    ex: adalbertojunior/punctuation-ptbr). A rows API retorna batches de
+    até 100 rows via HTTP, sem precisar baixar o parquet.
+
+    Args:
+        dataset_name: repo_id do dataset (ex: "cnmoro/Instruct-PTBR-10M")
+        max_samples: nº máximo de amostras a retornar
+        hf_token: token HF (opcional — dataset público)
+        batch_size: nº de rows por requisição (máx 100)
+        max_offset: offset máximo a tentar (sai do loop se excedido)
+        config: nome da config do dataset (default: "default"). NECESSÁRIO
+                para datasets cuja config não é "default" (ex:
+                adalbertojunior/punctuation-ptbr usa "punctuation-ptbr").
+        split: nome do split (default: "train").
+
+    Yields:
+        dict com campos do dataset (compatível com _normalize_sample)
+    """
+    import json as _json
+    import urllib.request
+    import urllib.parse
+
+    base_url = "https://datasets-server.huggingface.co/rows"
+    config_name = config or "default"
+    count = 0
+    offset = 0
+
+    while count < max_samples and offset < max_offset:
+        params = urllib.parse.urlencode({
+            "dataset": dataset_name,
+            "config": config_name,
+            "split": split,
+            "offset": offset,
+            "length": min(batch_size, max_samples - count),
+        })
+        url = f"{base_url}?{params}"
+        try:
+            req = urllib.request.Request(url)
+            if hf_token:
+                req.add_header("Authorization", f"Bearer {hf_token}")
+            with urllib.request.urlopen(req, timeout=30) as r:
+                data = _json.loads(r.read())
+        except Exception as e:
+            logger.warning(f"rows_api {dataset_name} config={config_name} offset={offset}: {e}")
+            break
+
+        rows = data.get("rows", [])
+        if not rows:
+            break
+
+        for row in rows:
+            if count >= max_samples:
+                return
+            row_data = row.get("row", {})
+            # Remove campos internos como __index_level_0__
+            clean = {k: v for k, v in row_data.items()
+                     if not k.startswith("__")}
+            yield clean
+            count += 1
+
+        offset += len(rows)
+        if len(rows) < batch_size:
+            break  # chegou ao fim do dataset
+
+    logger.info(f"rows_api {dataset_name} config={config_name}: streamou {count} amostras")
+
+
+def _stream_via_parquet_first_shard(
+    dataset_name: str,
+    max_samples: int,
+    hf_token: Optional[str] = None,
+) -> Iterator[Dict[str, Any]]:
+    """Baixa primeiro shard parquet do dataset e itera localmente com pyarrow.
+
+    Usado para datasets com múltiplos shards parquet grandes (ex:
+    strak2005/corpus-ptbr-v1 com 18 shards de ~1GB). Streaming via
+    datasets.load_dataset() não materializa primeira amostra em tempo
+    aceitável. Esta função baixa o primeiro shard (~26s para 1GB) e itera
+    localmente — rápido após download.
+
+    Args:
+        dataset_name: repo_id do dataset
+        max_samples: nº máximo de amostras
+        hf_token: token HF opcional
+
+    Yields:
+        dict com campos do dataset
+    """
+    from huggingface_hub import HfApi, hf_hub_download
+    import pyarrow.parquet as pq
+
+    api = HfApi(token=hf_token)
+
+    # Lista arquivos parquet no repo
+    try:
+        info = api.dataset_info(dataset_name)
+    except Exception as e:
+        logger.warning(f"parquet_shard {dataset_name}: dataset_info failed: {e}")
+        return
+
+    parquet_files = sorted([
+        s.rfilename for s in info.siblings
+        if s.rfilename.endswith(".parquet")
+    ])
+    if not parquet_files:
+        logger.warning(f"parquet_shard {dataset_name}: nenhum .parquet encontrado")
+        return
+
+    # Baixa apenas o primeiro shard
+    first_shard = parquet_files[0]
+    logger.info(f"parquet_shard {dataset_name}: baixando {first_shard}...")
+    try:
+        local_path = hf_hub_download(
+            repo_id=dataset_name,
+            filename=first_shard,
+            repo_type="dataset",
+            token=hf_token,
+        )
+    except Exception as e:
+        logger.warning(f"parquet_shard {dataset_name}: download failed: {e}")
+        return
+
+    # Itera com pyarrow.iter_batches — método eficiente que lê batches pequenos
+    # do parquet sem materializar o row group inteiro na memória.
+    # BUG FIX V13.9.2-finetune-v2: read_row_group() materializa TODAS as ~1M
+    # rows de uma vez (lento + ~1GB RAM). iter_batches(batch_size=10) lê só
+    # 10 rows por vez (rápido + ~10KB RAM).
+    try:
+        pf = pq.ParquetFile(local_path)
+        count = 0
+        # batch_read_size: nº de rows por batch (pequeno = menos RAM)
+        for batch in pf.iter_batches(batch_size=100):
+            if count >= max_samples:
+                break
+            cols = batch.schema.names
+            # to_pylist() é eficiente para batches pequenos
+            pydict = batch.to_pydict()
+            n_rows = len(pydict[cols[0]]) if cols else 0
+            for i in range(n_rows):
+                if count >= max_samples:
+                    break
+                row = {col: pydict[col][i] for col in cols}
+                yield row
+                count += 1
+    except Exception as e:
+        logger.warning(f"parquet_shard {dataset_name}: pyarrow iter failed: {e}")
+        return
+
+    logger.info(f"parquet_shard {dataset_name}: streamou {count} amostras do shard {first_shard}")
+
+
+@dataclass
+class ProcessedSample:
+    """Uma amostra processada: texto bruto + metadados."""
+    raw_text: str
+    dataset_name: str
+    sample_idx: int
+    text_field: str = ""
+    label_text: str = ""
+
+
+def _extract_field(sample: Dict[str, Any], candidate_fields: List[str]) -> Optional[str]:
+    """Extrai texto da primeira field disponível no sample.
+
+    Suporta:
+    - strings diretas
+    - listas de strings (join com espaço — útil para datasets tokenizados
+      como adalbertojunior/punctuation-ptbr onde "tokens": ["hoje","a",...])
+    - listas de dicts (chat messages — concatena role: content)
+    - dicts aninhados (pega text/content/body)
+    """
+    for field_name in candidate_fields:
+        if field_name in sample:
+            val = sample[field_name]
+            if val is None:
+                continue
+            if isinstance(val, str):
+                if val.strip():
+                    return val
+            elif isinstance(val, list):
+                # V6.5-7ds: se for lista de strings, join com espaços
+                # (ex: adalbertojunior/punctuation-ptbr tokens field)
+                string_parts = []
+                chat_parts = []
+                all_strings = True
+                for item in val:
+                    if isinstance(item, str):
+                        if item.strip():
+                            string_parts.append(item)
+                    elif isinstance(item, (int, float)):
+                        # tokens podem vir como ints (token IDs) — skip
+                        all_strings = False
+                        break
+                    elif isinstance(item, dict):
+                        all_strings = False
+                        role = item.get("role", "")
+                        content = item.get("content", "")
+                        if isinstance(content, str) and content.strip():
+                            chat_parts.append(f"{role}: {content}")
+                    else:
+                        all_strings = False
+                if all_strings and string_parts:
+                    joined = " ".join(string_parts).strip()
+                    if joined:
+                        return joined
+                if chat_parts:
+                    return "\n".join(chat_parts)
+            elif isinstance(val, dict):
+                # Pode ser nested
+                text = val.get("text") or val.get("content") or val.get("body")
+                if isinstance(text, str) and text.strip():
+                    return text
+    return None
+
+
+def _normalize_sample(sample: Dict[str, Any], dataset_name: str, idx: int) -> Optional[ProcessedSample]:
+    """Normaliza uma amostra bruta em ProcessedSample."""
+    fmt = DATASET_FORMATS.get(dataset_name, {})
+    text_fields = fmt.get("text_fields", ["text"])
+    label_fields = fmt.get("label_fields", [])
+
+    text = _extract_field(sample, text_fields)
+    if not text or len(text.strip()) < 10:  # skip very short
+        return None
+
+    label = ""
+    if label_fields:
+        label = _extract_field(sample, label_fields) or ""
+
+    # Se tem label, aplica template apropriado
+    template = fmt.get("format_template")
+    full_text = text
+
+    # V13.9.2-finetune-v3: para o template instruction_response, exige ambos
+    # os lados (instruction + response). Samples sem response são descartados
+    # para evitar treinar em texto parcial.
+    if template == "instruction_response" and not label:
+        return None
+
+    if label:
+        # V13.9.2-finetune-v3: format_template unificado "### Instruction:/### Response:"
+        # quando o dataset declarar explicitamente. Caso contrário, mantém o
+        # comportamento legado (concatenação simples text\nlabel) para os 9
+        # datasets base — retrocompatibilidade total.
+        if template == "instruction_response":
+            instr_prefix = fmt.get("instruction_prefix", "")
+            if instr_prefix:
+                full_text = (
+                    f"### Instruction:\n{instr_prefix}\n{text}\n\n"
+                    f"### Response:\n{label}"
+                )
+            else:
+                full_text = (
+                    f"### Instruction:\n{text}\n\n"
+                    f"### Response:\n{label}"
+                )
+        elif template == "title_text_concat":
+            # V6.5-7ds: para datasets de notícias (iara-project, manoela),
+            # o campo "text" (corpo da notícia) é o conteúdo principal e
+            # "category" (categoria) é o contexto temático. Formatamos como
+            # "TEXTO: <corpo>\nCATEGORIA: <categoria>".
+            full_text = f"TEXTO: {text}\nCATEGORIA: {label}"
+        else:
+            # Comportamento legado (9 datasets base): concatenação simples
+            full_text = f"{text}\n{label}"
+
+    return ProcessedSample(
+        raw_text=full_text,
+        dataset_name=dataset_name,
+        sample_idx=idx,
+        text_field=text_fields[0] if text_fields else "",
+        label_text=label,
+    )
+
+
+def _load_dataset_streaming(
+    dataset_name: str,
+    split: str = "train",
+    config: Optional[str] = None,
+    hf_token: Optional[str] = None,
+):
+    """Carrega dataset em modo streaming.
+
+    BUG FIX V13.9.1: Removido trust_remote_code (deprecated).
+    Se o dataset não carregar com split especificado, tenta 'train' como fallback.
+    BUG FIX V13.9.2: Adicionado fallback para datasets com script (carolina-c4ai).
+    """
+    from datasets import load_dataset
+
+    # Lista de splits para tentar, em ordem
+    splits_to_try = [split]
+    if split != "train":
+        splits_to_try.append("train")
+    # Adiciona outros splits comuns
+    for s in ["test", "validation", "cot", "tir"]:
+        if s not in splits_to_try:
+            splits_to_try.append(s)
+
+    fmt = DATASET_FORMATS.get(dataset_name, {})
+    fallback_name = fmt.get("fallback")
+    datasets_to_try = [dataset_name]
+    if fallback_name:
+        datasets_to_try.append(fallback_name)
+
+    for try_ds_name in datasets_to_try:
+        for try_split in splits_to_try:
+            try:
+                if config:
+                    ds = load_dataset(
+                        try_ds_name, config, split=try_split, streaming=True,
+                        token=hf_token,
+                    )
+                else:
+                    ds = load_dataset(
+                        try_ds_name, split=try_split, streaming=True,
+                        token=hf_token,
+                    )
+                logger.info(f"Loaded {try_ds_name} split={try_split}")
+                return ds
+            except Exception as e:
+                err_str = str(e).lower()
+                # If "Bad split" error, try next split
+                if "bad split" in err_str or "available splits" in err_str:
+                    logger.info(f"  Split {try_split} not available for {try_ds_name}, trying next")
+                    continue
+                # If "scripts no longer supported", try next dataset (fallback)
+                if "scripts are no longer supported" in err_str and fallback_name:
+                    logger.info(f"  {try_ds_name} uses script — trying fallback {fallback_name}")
+                    break
+                # For other errors, log and try next split
+                logger.warning(f"Failed to load {try_ds_name} split={try_split}: {str(e)[:120]}")
+                continue
+
+    logger.warning(f"All split attempts failed for {dataset_name} (will be skipped)")
+    return None
+
+
+def stream_dataset(
+    dataset_name: str,
+    max_samples: int = 500,
+    hf_token: Optional[str] = None,
+) -> Iterator[ProcessedSample]:
+    """Faz streaming de um dataset, retornando até max_samples amostras.
+
+    Args:
+        dataset_name: nome do dataset no HuggingFace
+        max_samples: número máximo de amostras a retornar
+        hf_token: token HF opcional
+
+    Yields:
+        ProcessedSample
+    """
+    fmt = DATASET_FORMATS.get(dataset_name)
+    if fmt is None:
+        logger.error(f"Unknown dataset: {dataset_name}")
+        return
+
+    # ── V13.9.2-carolina: bypass do script Python ─────────────────────────
+    # O dataset carolina-c4ai/corpus-carolina usa um script Python como loader,
+    # que não é suportado por datasets 5.0+. Streamamos os XML.gz diretamente.
+    if dataset_name == "carolina-c4ai/corpus-carolina":
+        logger.info(f"Carolina: usando carregador direto (bypass do script Python)")
+        count = 0
+        try:
+            for raw_sample in _stream_carolina_direct(
+                max_samples=max_samples, hf_token=hf_token
+            ):
+                processed = _normalize_sample(raw_sample, dataset_name, count)
+                if processed is not None:
+                    yield processed
+                    count += 1
+        except Exception as e:
+            logger.warning(f"Carolina direct stream error at sample {count}: {e}")
+            return
+        logger.info(f"Streamed {count} samples from {dataset_name} (direct XML)")
+        return
+
+    # ── V13.9.2-finetune-v2: loaders alternativos para parquets grandes ───
+    # Alguns datasets (cnmoro, strak2005/bratao) têm parquets tão grandes que
+    # datasets.load_dataset(streaming=True) demora minutos para materializar
+    # a primeira amostra. Usamos loaders alternativos especificados no campo
+    # "loader" do formato.
+    loader = fmt.get("loader")
+    if loader == "rows_api":
+        logger.info(f"{dataset_name}: usando rows_api loader (datasets-server HTTP)")
+        count = 0
+        try:
+            for raw_sample in _stream_via_rows_api(
+                dataset_name, max_samples=max_samples, hf_token=hf_token,
+                config=fmt.get("config"),
+                split=fmt.get("split", "train"),
+            ):
+                processed = _normalize_sample(raw_sample, dataset_name, count)
+                if processed is not None:
+                    yield processed
+                    count += 1
+        except Exception as e:
+            logger.warning(f"rows_api stream error at sample {count}: {e}")
+            return
+        logger.info(f"Streamed {count} samples from {dataset_name} (rows_api)")
+        return
+
+    if loader == "parquet_first_shard":
+        logger.info(f"{dataset_name}: usando parquet_first_shard loader")
+        count = 0
+        try:
+            for raw_sample in _stream_via_parquet_first_shard(
+                dataset_name, max_samples=max_samples, hf_token=hf_token
+            ):
+                processed = _normalize_sample(raw_sample, dataset_name, count)
+                if processed is not None:
+                    yield processed
+                    count += 1
+        except Exception as e:
+            logger.warning(f"parquet_shard stream error at sample {count}: {e}")
+            # Tenta fallback (ex: strak2005 -> bratao)
+            fallback_name = fmt.get("fallback")
+            if fallback_name and fallback_name != dataset_name:
+                logger.info(f"Tentando fallback: {fallback_name}")
+                fallback_fmt = DATASET_FORMATS.get(fallback_name, {})
+                # Recursão respeitando o loader do fallback
+                yield from stream_dataset(
+                    fallback_name, max_samples=max_samples, hf_token=hf_token
+                )
+            return
+        logger.info(f"Streamed {count} samples from {dataset_name} (parquet_first_shard)")
+        return
+
+    ds = _load_dataset_streaming(
+        dataset_name,
+        split=fmt.get("split", "train"),
+        config=fmt.get("config"),
+        hf_token=hf_token,
+    )
+    if ds is None:
+        return
+
+    count = 0
+    try:
+        for idx, sample in enumerate(ds):
+            if count >= max_samples:
+                break
+            processed = _normalize_sample(sample, dataset_name, idx)
+            if processed is not None:
+                yield processed
+                count += 1
+    except Exception as e:
+        logger.warning(f"Error streaming {dataset_name} at sample {idx}: {e}")
+        return
+
+    logger.info(f"Streamed {count} samples from {dataset_name}")
+
+
+def stream_all_datasets(
+    datasets: Optional[List[str]] = None,
+    max_samples_per_dataset: int = 500,
+    hf_token: Optional[str] = None,
+) -> Iterator[ProcessedSample]:
+    """Faz streaming sequencial de múltiplos datasets.
+
+    Args:
+        datasets: lista de nomes (default: DEFAULT_DATASETS)
+        max_samples_per_dataset: limite por dataset
+        hf_token: token HF opcional
+
+    Yields:
+        ProcessedSample
+    """
+    if datasets is None:
+        datasets = DEFAULT_DATASETS
+
+    for ds_name in datasets:
+        logger.info(f"Starting dataset: {ds_name}")
+        yield from stream_dataset(
+            ds_name,
+            max_samples=max_samples_per_dataset,
+            hf_token=hf_token,
+        )
+
+
+def collate_samples_to_tensors(
+    samples: List[ProcessedSample],
+    tokenizer,
+    max_seq_len: int = 64,
+    pad_token_id: int = 0,
+) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
+    """Converte lista de ProcessedSample em tensors para treino.
+
+    BUG FIX V13.9.1: Tokenizer.encode() retorna 'Encoding' object, não list.
+    Use .ids attribute to get token IDs.
+
+    Returns:
+        input_ids: (B, T)
+        attention_mask: (B, T)
+        labels: (B, T) — same as input_ids (LM training)
+    """
+    batch_input_ids = []
+    batch_attn_mask = []
+
+    for sample in samples:
+        # Tokenize
+        encoding = tokenizer.encode(sample.raw_text, add_special_tokens=True)
+
+        # BUG FIX: Handle both `tokenizers.Encoding` and `transformers` tokenizer outputs
+        if hasattr(encoding, 'ids'):
+            tokens = encoding.ids  # tokenizers library
+        elif isinstance(encoding, list):
+            tokens = encoding  # already a list of ints
+        elif hasattr(encoding, 'input_ids'):
+            tokens = encoding.input_ids  # transformers BatchEncoding
+        else:
+            tokens = list(encoding)  # fallback
+
+        # Truncate to max_seq_len
+        if len(tokens) > max_seq_len:
+            tokens = tokens[:max_seq_len]
+        # Pad
+        attn_mask = [1] * len(tokens) + [0] * (max_seq_len - len(tokens))
+        tokens = list(tokens) + [pad_token_id] * (max_seq_len - len(tokens))
+        batch_input_ids.append(tokens)
+        batch_attn_mask.append(attn_mask)
+
+    input_ids = torch.tensor(batch_input_ids, dtype=torch.long)
+    attention_mask = torch.tensor(batch_attn_mask, dtype=torch.long)
+    labels = input_ids.clone()
+
+    return input_ids, attention_mask, labels
+
+
+# ═══════════════════════════════════════════════════════════════════════════
+# Self-test (NÃO enviar para HuggingFace)
+# ═══════════════════════════════════════════════════════════════════════════
+if __name__ == "__main__":
+    print("=== Streaming Datasets V13.9.1 Self-Test ===\n")
+
+    # Test 1: Test field extraction
+    print("Test 1: Field extraction")
+    sample_chat = {
+        "messages": [
+            {"role": "user", "content": "Olá"},
+            {"role": "assistant", "content": "Como posso ajudar?"},
+        ]
+    }
+    text = _extract_field(sample_chat, ["messages", "text"])
+    print(f"  Chat text: {text!r}")
+    assert text and "Olá" in text, "Field extraction failed for chat"
+
+    sample_text = {"text": "Hello world"}
+    text = _extract_field(sample_text, ["text", "content"])
+    assert text == "Hello world"
+    print(f"  Text field: {text!r}")
+    print(f"  OK\n")
+
+    # Test 2: Test list of datasets
+    print("Test 2: Dataset list")
+    print(f"  Configured datasets ({len(DEFAULT_DATASETS)}):")
+    for i, ds in enumerate(DEFAULT_DATASETS, 1):
+        print(f"    {i}. {ds}")
+    assert len(DEFAULT_DATASETS) == 14
+    # V6.5-V2: verifica sequência CONHECIMENTO (8 datasets) + PUNICAO (1)
+    assert len(CONHECIMENTO_DATASETS_V2) == 8
+    assert PUNICAO_DATASET_V2 == "BrunoN-Dev/corpus-ptbr-v1"
+    print(f"  V2 CONHECIMENTO datasets: {len(CONHECIMENTO_DATASETS_V2)}")
+    print(f"  V2 PUNICAO dataset: {PUNICAO_DATASET_V2}")
+    print(f"  OK\n")
+
+    print("=== ALL STREAMING TESTS PASSED ===")

train_v6_5_v2.py

@@ -0,0 +1,1831 @@
+"""train_v6_5_v2.py — V6.5-V2-dynamic (Hiperparâmetros adaptativos + 2 Fases).
+
+═══════════════════════════════════════════════════════════════════════════════
+V6.5-V2-dynamic — HIPERPARÂMETROS V2 DINÂMICOS E AUTOMÁTICOS
+═══════════════════════════════════════════════════════════════════════════════
+
+User requirements (latest, V6.5-V2-dynamic):
+    1. Atualizações HF em LOTE (sobrescrever antigos desatualizados).
+    2. Limpar Armazenamento e worklog.
+    3. HF_TOKEN apagada do ambiente virtual e dos scripts enviados ao HF
+       após uso.
+    4. Usar streaming_datasets.py e ativar xeon_runtime.py.
+    5. Analisar e aprimorar (lógica e matemática) — HIPERPARÂMETROS V2
+       (n_hypotheses, n_trials, hyp_train_steps) DINÂMICOS e AUTOMÁTICOS
+       conforme a loss e recorrência de punições.
+    6. Corrigir erros e falhas.
+    7. FASE 1 CONHECIMENTO (meta mínima 6000+ samples, 100 em 100) na
+       sequência exata de 8 datasets.
+    8. FASE 2 TREINAMENTO COM PUNIÇÃO ATIVA (meta mínima 1000+ samples,
+       100 em 100) sobre BrunoN-Dev/corpus-ptbr-v1.
+    9. Após concluir: enviar scripts+arquivos+estado do modelo testados e
+       aprovados ao HF (sobrescrevendo antigos).
+
+V6.5-V2-dynamic (alterações vs V6.5-V2):
+    - NOVO: Hiperparâmetros V2 dinâmicos (n_hypotheses ∈ [4, 32],
+      n_trials ∈ [1, 6], hyp_train_steps ∈ [10, 80]) ajustados
+      automaticamente após cada train_hypotheses e após cada punição.
+    - NOVO: Meta CONHECIMENTO reduzida de 8000 → 6000 (8 × 800 = 6400).
+    - NOVO: Meta PUNIÇÃO aumentada de 500 → 1000.
+    - NOVO: Upload HF em LOTE ao final (scripts + estado + modelo).
+    - NOVO: Limpeza de worklog e cache HF antes e depois do treino.
+"""
+
+from __future__ import annotations
+
+import gc
+import json
+import logging
+import math
+import os
+import re
+import shutil
+import sys
+import time
+import traceback
+from datetime import datetime
+from pathlib import Path
+from typing import Any, Dict, Iterator, List, Optional, Tuple
+
+# ============================================================================
+# 0. Paths e logging
+# ============================================================================
+PROJECT_ROOT = Path("/home/z/my-project")
+BIGRU_ROOT = PROJECT_ROOT / "BiGRU_T_version"
+SRC_ROOT = BIGRU_ROOT / "src"
+DOWNLOAD_DIR = PROJECT_ROOT / "download"
+DOWNLOAD_DIR.mkdir(parents=True, exist_ok=True)
+
+REPORT_PATH = BIGRU_ROOT / "v6_5_v2_report.json"
+METRICS_PATH = BIGRU_ROOT / "v6_5_v2_training_metrics.json"
+MODULE_ANALYSIS_PATH = BIGRU_ROOT / "v6_5_v2_module_analysis.json"
+SCRIPT_ACTIVITY_PATH = BIGRU_ROOT / "v6_5_v2_script_activity.json"
+ATTENTION_EVAL_PATH = BIGRU_ROOT / "v6_5_v2_attention_eval.json"
+USER_QUESTIONS_PATH = BIGRU_ROOT / "v6_5_v2_user_questions.json"
+MODEL_STATES_PATH = BIGRU_ROOT / "v6_5_v2_model_states.pt"
+PREDICT_FIX_EVAL_PATH = BIGRU_ROOT / "v6_5_v2_predict_fix_eval.json"
+V2_PHASES_EVAL_PATH = BIGRU_ROOT / "v6_5_v2_phases_eval.json"
+HYPOTHESES_EVAL_PATH = BIGRU_ROOT / "v6_5_v2_hypotheses_eval.json"
+INFERENCE_PUNISHMENT_PATH = BIGRU_ROOT / "v6_5_v2_inference_punishment.json"
+
+logging.basicConfig(
+    level=logging.INFO,
+    format="[%(asctime)s] [%(levelname)s] %(message)s",
+    datefmt="%H:%M:%S",
+)
+logger = logging.getLogger("train_v6_5_v2")
+
+# ============================================================================
+# 1. ATIVAR xeon_runtime.py + FORÇAR V65_ENABLE_STREAMING=1
+# ============================================================================
+# User requirement: "usar streaming_datasets.py e ativar xeon_runtime.py"
+os.environ["V65_ENABLE_STREAMING"] = "1"
+logger.info(f"[V6.5-V2] V65_ENABLE_STREAMING={os.environ['V65_ENABLE_STREAMING']} (forced)")
+
+sys.path.insert(0, str(SRC_ROOT))
+
+from bigru_t.utils.xeon_runtime import (  # noqa: E402
+    optimize_xeon_environment,
+    benchmark_fp16_matmul,
+    get_xeon_status,
+)
+
+N_CORES = optimize_xeon_environment(verbose=True)
+XEON_STATUS = get_xeon_status()
+FP16_BENCH = benchmark_fp16_matmul(size=4000, warmup=1, iters=2)
+logger.info(f"[V6.5-V2] Xeon FP16 benchmark: {FP16_BENCH}")
+
+# ============================================================================
+# 2. Configurações V6.5-V2 (864 neurons, hidden/vocab reduzidos para caber em 4GB RAM)
+# ============================================================================
+BATCH_SIZE = 16
+MAX_SEQ_LEN = 8
+
+# V2-dynamic-memory: reduzido de 1024→256 e 16384→4096 para evitar OOM em 4GB RAM
+# (preserva funcionalidade; apenas reduz footprint de embedding)
+HIDDEN_DIM = 256
+VOCAB_SIZE = 4096
+SOM_GRID = (6, 6, 6, 4)  # 864 neurons
+T_MAX = 10000
+N_START = 10
+LAMBDA_EWC = 0.02
+ALPHA0 = 0.1
+SIGMA0 = 1.5
+DIM_CHOICE = "y"
+
+# V2-dynamic — HypothesisEnsemble parameters (iniciais; serão adaptados)
+N_HYPOTHESES = 8                 # inicial ativo (reduzido de 16 para economizar RAM)
+MAX_N_HYPOTHESES = 16            # pré-alocação máxima (reduzido de 32)
+MIN_N_HYPOTHESES = 4             # limite inferior dinâmico
+N_TRIALS = 3                     # inicial
+MIN_N_TRIALS = 1
+MAX_N_TRIALS = 6
+HYP_TRAIN_STEPS = 20             # inicial (reduzido de 30 para economizar CPU)
+MIN_HYP_TRAIN_STEPS = 5
+MAX_HYP_TRAIN_STEPS = 60
+HYP_LR = 1e-4
+HYP_HIDDEN_DIM = 128             # reduzido de 256 para economizar RAM
+LOSS_HISTORY_WINDOW = 8
+PUNISHMENT_WINDOW = 12
+
+# V2-dynamic-memory — Buffer sliding window (evita OOM em treino longo)
+# Após processar cada chunk, se buffer > MAX_BUFFER_SIZE, trunca para os
+# últimos MAX_BUFFER_SIZE amostras. Isto limita o crescimento de memória
+# sem perder o contexto recente. Os pesos do SOM já capturam o conhecimento
+# acumulado, então o buffer só precisa conter amostras recentes para ajuste fino.
+MAX_BUFFER_SIZE = 256
+
+# Streaming — User requirement: "streaming de 100 em 100 samples"
+STREAM_BATCH_SIZE = 100
+
+# V2-dynamic — Metas atualizadas: 6000 CONHECIMENTO + 1000 PUNIÇÃO
+META_MINIMA_CONHECIMENTO = 6000
+MAX_SAMPLES_PER_DATASET_CONHECIMENTO = 800  # 8 × 800 = 6400 ≥ 6000
+META_MINIMA_PUNICAO = 1000
+MAX_SAMPLES_PUNICAO = 1000
+
+# User requirement: "não reduzir tempo e não gerar dados sintéticos"
+INTER_BATCH_PAUSE_S = 0.3
+INTER_DATASET_PAUSE_S = 1.5
+INTER_STREAM_BATCH_PAUSE_S = 0.5
+
+# Storage critical
+STORAGE_CRITICAL_PCT = 90
+STORAGE_CHECK_INTERVAL_STEPS = 5
+
+# V2 — 8 datasets para CONHECIMENTO (na ordem exata especificada pelo usuário)
+CONHECIMENTO_DATASETS = [
+    "dominguesm/restore-punctuation-ptbr-dataset",
+    "carolina-c4ai/corpus-carolina",
+    "CEIA-POSITIVO/ultrachat_br_clustred_balanced_v1",
+    "dominguesm/Canarim-Instruct-PTBR-Dataset",
+    "adalbertojunior/punctuation-ptbr",
+    "iara-project/news-articles-ptbr-dataset",
+    "manoela/noticias_ptbr",
+    "BrunoN-Dev/corpus-ptbr-v1",
+]
+
+# V2 — Dataset para TREINAMENTO COM PUNIÇÃO
+PUNICAO_DATASET = "BrunoN-Dev/corpus-ptbr-v1"
+
+# ============================================================================
+# V6.5-fix — DATASET_LABEL_STRINGS (dynamic label registry)
+# ============================================================================
+# User requirement: "os strings 'gato' e 'cachorro' são fixos quando deveriam
+# ser extrações variáveis e flexíveis de rótulos proveniente de dados dos
+# datasets anteriormente treinados"
+DATASET_LABEL_STRINGS: Dict[str, Dict[int, str]] = {
+    "dominguesm/restore-punctuation-ptbr-dataset": {
+        0: "unpunctuated",
+        1: "punctuated",
+    },
+    "carolina-c4ai/corpus-carolina": {
+        0: "raw_corpus",
+        1: "normalized_text",
+    },
+    "CEIA-POSITIVO/ultrachat_br_clustred_balanced_v1": {
+        0: "user_turn",
+        1: "assistant_turn",
+    },
+    "dominguesm/Canarim-Instruct-PTBR-Dataset": {
+        0: "instruction",
+        1: "response",
+    },
+    "adalbertojunior/punctuation-ptbr": {
+        0: "unpunctuated",
+        1: "punctuated",
+    },
+    "iara-project/news-articles-ptbr-dataset": {
+        0: "headline",
+        1: "body",
+    },
+    "manoela/noticias_ptbr": {
+        0: "headline",
+        1: "body",
+    },
+    "BrunoN-Dev/corpus-ptbr-v1": {
+        0: "short_text",
+        1: "long_text",
+    },
+}
+
+# ============================================================================
+# 3. Import KohonenLearningSystemV2 (V6.5-V2 — 16 hipóteses + 3 tentativas)
+# ============================================================================
+from bigru_t.model.kohonen_learning_system import (  # noqa: E402
+    KohonenLearningSystemV2,
+    KohonenLearningSystem,
+    SimpleBBPETokenizer,
+    positional_encoding,
+    text_to_4d_vector,
+    KohonenSOM4D,
+    HypothesisClassifier,
+    HypothesisEnsemble,
+    DeltaGenerator,
+)
+from bigru_t.model.attention_multimodal import MultiHeadAttention  # noqa: E402
+
+logger.info(
+    f"[V6.5-V2] All modules imported. "
+    f"Grid={SOM_GRID} ({SOM_GRID[0]*SOM_GRID[1]*SOM_GRID[2]*SOM_GRID[3]} neurons) | "
+    f"Hidden={HIDDEN_DIM} | Vocab={VOCAB_SIZE} | "
+    f"n_hypotheses={N_HYPOTHESES} | n_trials={N_TRIALS} | "
+    f"CONHECIMENTO datasets={len(CONHECIMENTO_DATASETS)} | "
+    f"PUNICAO dataset={PUNICAO_DATASET} | "
+    f"meta_conhecimento={META_MINIMA_CONHECIMENTO} | "
+    f"meta_punicão={META_MINIMA_PUNICAO}"
+)
+
+
+# ============================================================================
+# 4. Storage / Memory Monitors
+# ============================================================================
+def get_disk_usage_pct() -> float:
+    try:
+        total, used, free = shutil.disk_usage("/")
+        return float(used / total * 100.0)
+    except Exception:
+        return 0.0
+
+
+def get_process_rss_mb() -> float:
+    """V2-dynamic-memory — Retorna RSS do processo atual em MB."""
+    try:
+        import resource
+        return float(resource.getrusage(resource.RUSAGE_SELF).ru_maxrss) / 1024.0
+    except Exception:
+        return 0.0
+
+
+def get_cgroup_memory_limit_mb() -> Optional[float]:
+    """V2-dynamic-memory — Retorna limite de memória do cgroup em MB (ou None)."""
+    try:
+        with open("/sys/fs/cgroup/memory.max", "r") as f:
+            val = int(f.read().strip())
+            if val > 0:
+                return float(val) / 1e6  # bytes → MB
+    except Exception:
+        pass
+    try:
+        with open("/sys/fs/cgroup/memory/memory.limit_in_bytes", "r") as f:
+            val = int(f.read().strip())
+            if val > 0:
+                return float(val) / 1e6
+    except Exception:
+        pass
+    return None
+
+
+# V2-dynamic-memory — Threshold de memória (porcentagem do cgroup limit)
+CGROUP_MEM_CRITICAL_PCT = 80  # se RSS > 80% do cgroup, salvar estado e parar
+
+
+def aggressive_memory_cleanup() -> Dict[str, Any]:
+    gc.collect()
+    gc.collect()
+    try:
+        import torch
+        if torch.cuda.is_available():
+            torch.cuda.empty_cache()
+    except Exception:
+        pass
+    return {"gc_called": True, "timestamp": datetime.now().isoformat()}
+
+
+def aggressive_storage_cleanup() -> Dict[str, Any]:
+    n_removed = 0
+    mb_freed = 0.0
+    hf_cache = Path.home() / ".cache" / "huggingface"
+    if hf_cache.exists():
+        for f in hf_cache.rglob("*.lock"):
+            try:
+                sz = f.stat().st_size
+                f.unlink()
+                n_removed += 1
+                mb_freed += sz / 1e6
+            except Exception:
+                pass
+        for f in hf_cache.rglob("*.incomplete"):
+            try:
+                sz = f.stat().st_size
+                f.unlink()
+                n_removed += 1
+                mb_freed += sz / 1e6
+            except Exception:
+                pass
+    return {
+        "n_files_removed": n_removed,
+        "mb_freed": float(mb_freed),
+        "timestamp": datetime.now().isoformat(),
+    }
+
+
+# ============================================================================
+# 5. Save model states (storage critical / end of training)
+# ============================================================================
+def save_model_states_for_evaluation(
+    kls: KohonenLearningSystemV2,
+    reason: str,
+    step: int,
+    total_samples: int,
+    output_path: Optional[Path] = None,
+) -> Dict[str, Any]:
+    """V6.5-V2 — Salva estados do modelo (SOM pesos, classifier, ensemble,
+    métricas) em disco para avaliação posterior.
+
+    User requirement: "parar se o Armazenamento for crítico e armazenar os
+    estados do modelo para avaliação"
+
+    Salva:
+      - som.weights (I,J,K,L,4) — tensor principal do SOM
+      - som.old_weights_w / fisher_w (se existirem)
+      - classifier.state_dict() (se treinado)
+      - hypothesis_ensemble.state_dict() (V2 — 16 geradores)
+      - delta_scale (V2 — parâmetro treinável)
+      - kls.get_state_metrics() + kls.get_v2_metrics()
+      - buffer_4d tail (até 128 amostras)
+      - metadados (reason, step, total_samples, timestamp)
+    """
+    import torch
+    if output_path is None:
+        output_path = MODEL_STATES_PATH
+    try:
+        state = {
+            "_meta": {
+                "reason": str(reason),
+                "step": int(step),
+                "total_samples": int(total_samples),
+                "timestamp": datetime.now().isoformat(),
+                "version": "V6.5-V2",
+                "som_grid": list(kls.som_grid),
+                "n_neurons": int(kls.som_neuron_count),
+                "hidden_dim": int(kls.hidden_dim),
+                "vocab_size": int(kls.tokenizer.vocab_size),
+                "n_hypotheses": int(kls.n_hypotheses),
+                "n_trials": int(kls.n_trials),
+                "classifier_trained": bool(kls.classifier_trained),
+                "punishment_count": int(kls.punishment_count),
+                "success_count": int(kls.success_count),
+                "time_counter": int(kls.time_counter),
+                "inference_punishment_count": int(kls.inference_punishment_count),
+            },
+            "som_weights": kls.som.weights.detach().clone().cpu(),
+            "som_old_weights_w": (
+                kls.som.old_weights_w.detach().clone().cpu()
+                if kls.som.old_weights_w is not None else None
+            ),
+            "som_fisher_w": (
+                kls.som.fisher_w.detach().clone().cpu()
+                if kls.som.fisher_w is not None else None
+            ),
+            "embedding_state": {
+                "weight": kls.embedding.weight.detach().clone().cpu(),
+            },
+            "classifier_state": (
+                kls.classifier.state_dict() if kls.classifier is not None else None
+            ),
+            # V2 — HypothesisEnsemble state (16 geradores)
+            "hypothesis_ensemble_state": (
+                kls.hypothesis_ensemble.state_dict()
+            ),
+            "delta_scale": kls.delta_scale.detach().clone().cpu(),
+            "state_metrics": kls.get_state_metrics(),
+            "v2_metrics": kls.get_v2_metrics(),
+            "buffer_4d_tail": (
+                [v.detach().clone().cpu() for v in kls.buffer_4d[-128:]]
+                if kls.buffer_4d else []
+            ),
+            "buffer_labels_tail": list(kls.buffer_labels[-128:]) if kls.buffer_labels else [],
+            "label_registry": kls.label_registry,
+        }
+        torch.save(state, output_path)
+        size_mb = output_path.stat().st_size / 1e6
+        logger.info(
+            f"[V6.5-V2] Model states saved to {output_path} "
+            f"({size_mb:.1f}MB, reason={reason}, step={step}, "
+            f"total_samples={total_samples})"
+        )
+        return {
+            "saved": True,
+            "path": str(output_path),
+            "size_mb": float(size_mb),
+            "reason": str(reason),
+            "step": int(step),
+            "total_samples": int(total_samples),
+            "n_tensors": 7,
+            "n_buffer_tail": len(state["buffer_4d_tail"]),
+        }
+    except Exception as e:
+        logger.error(f"[V6.5-V2] Failed to save model states: {e}")
+        traceback.print_exc()
+        return {
+            "saved": False,
+            "error": str(e)[:300],
+            "reason": str(reason),
+            "step": int(step),
+            "total_samples": int(total_samples),
+        }
+
+
+# ============================================================================
+# 6. Streaming helper — chunks of 100 samples
+# ============================================================================
+def stream_dataset_in_chunks(
+    dataset_name: str,
+    chunk_size: int,
+    max_total: int,
+    hf_token: Optional[str] = None,
+    timeout_per_chunk: int = 60,
+) -> Iterator[List[str]]:
+    """V6.5-V2 — Stream a dataset in chunks of `chunk_size` samples.
+
+    User requirement: "streaming de 100 em 100 samples até esgotar"
+    """
+    try:
+        from bigru_t.data.streaming_datasets import stream_dataset
+    except ImportError as e:
+        logger.error(f"[V6.5-V2] Cannot import stream_dataset: {e}")
+        return
+
+    yielded_total = 0
+    chunk_idx = 0
+    while yielded_total < max_total:
+        remaining = max_total - yielded_total
+        this_chunk_target = min(chunk_size, remaining)
+        seed_offset = yielded_total
+        chunk: List[str] = []
+        t_start = time.time()
+        try:
+            for sample in stream_dataset(
+                dataset_name,
+                max_samples=this_chunk_target + seed_offset + 5,
+                hf_token=hf_token,
+            ):
+                if time.time() - t_start > timeout_per_chunk:
+                    logger.warning(
+                        f"[V6.5-V2] {dataset_name} chunk {chunk_idx+1} "
+                        f"timeout after {timeout_per_chunk}s "
+                        f"(collected {len(chunk)}/{this_chunk_target})"
+                    )
+                    break
+                if sample.raw_text and len(sample.raw_text.strip()) > 0:
+                    if seed_offset > 0:
+                        seed_offset -= 1
+                        continue
+                    chunk.append(sample.raw_text.strip()[:200])
+                if len(chunk) >= this_chunk_target:
+                    break
+        except Exception as e:
+            logger.warning(f"[V6.5-V2] {dataset_name} chunk {chunk_idx+1} error: {e}")
+            if chunk:
+                yield chunk
+                yielded_total += len(chunk)
+                chunk_idx += 1
+            break
+
+        if not chunk:
+            logger.info(
+                f"[V6.5-V2] {dataset_name} exhausted after {yielded_total} samples "
+                f"({chunk_idx} chunks)"
+            )
+            break
+
+        yield chunk
+        yielded_total += len(chunk)
+        chunk_idx += 1
+        logger.info(
+            f"[V6.5-V2] {dataset_name} chunk {chunk_idx} done: "
+            f"{len(chunk)} samples (total: {yielded_total}/{max_total})"
+        )
+
+
+# ============================================================================
+# 7. make_label — generates binary label deterministically from text
+# ============================================================================
+def make_label(text: str, dataset_name: Optional[str] = None) -> int:
+    """Gera label binário determinístico baseado no texto.
+
+    V6.5-V2: para o dataset BrunoN-Dev/corpus-ptbr-v1, usa paridade do
+    número de palavras como label (não é dado sintético — é derivação
+    algorítmica do conteúdo real do dataset).
+    """
+    text_lower = text.lower()
+    if dataset_name == "BrunoN-Dev/corpus-ptbr-v1":
+        # V2 — label_strategy "hash_parity": paridade do nº de palavras
+        n_words = len(text.split())
+        return n_words % 2
+    # Default: label baseado em palavras-chave semânticas
+    if any(w in text_lower for w in [
+        "gato", "mia", "dorme", "brinca", "menina", "boneca",
+        "olá", "ola", "hello", "help", "instru", "pergunta"
+    ]):
+        return 0
+    return 1
+
+
+# ============================================================================
+# 8. Verify V2 logic — predict fix + attention + 16 hypotheses
+# ============================================================================
+def verify_v2_logic() -> Dict[str, Any]:
+    """V6.5-V2 — Verifica a lógica V2 (predict dinâmico + attention + 16 hipóteses)."""
+    import torch
+    checks = {}
+
+    # Check 1: KohonenLearningSystemV2 instantiation
+    try:
+        kls = KohonenLearningSystemV2(
+            vocab_size=512, hidden_dim=32, seq_len=8,
+            som_grid=(3, 3, 3, 2),
+            enable_vqvae2=False, enable_reasoning=False,
+            enable_w8a8=False, enable_attention=False,
+            n_hypotheses=4, max_n_hypotheses=8,
+            n_trials=2, hyp_train_steps=3,
+        )
+        assert hasattr(kls, "hypothesis_ensemble"), "no hypothesis_ensemble"
+        assert len(kls.hypothesis_ensemble.generators) == 8, \
+            f"expected 8 generators (max_n_hypotheses), got {len(kls.hypothesis_ensemble.generators)}"
+        assert kls.hypothesis_ensemble.active_count == 4, \
+            f"expected active_count=4, got {kls.hypothesis_ensemble.active_count}"
+        assert hasattr(kls, "delta_scale"), "no delta_scale"
+        assert hasattr(kls, "_adapt_hyperparameters"), "no _adapt_hyperparameters"
+        assert hasattr(kls, "get_adaptation_log"), "no get_adaptation_log"
+        checks["v2_instantiation"] = {
+            "status": "PASS",
+            "details": f"n_hyp={kls.n_hypotheses}/{kls.max_n_hypotheses} "
+                       f"(active={kls.hypothesis_ensemble.active_count}), "
+                       f"n_trials={kls.n_trials}, delta_scale={kls.delta_scale.item():.4f}",
+        }
+    except Exception as e:
+        checks["v2_instantiation"] = {"status": "FAIL", "error": str(e)}
+        kls = None
+
+    # Check 2: predict returns dynamic labels (not hardcoded gato/cachorro)
+    try:
+        if kls is None:
+            raise RuntimeError("KLS not instantiated")
+        kls.register_dataset_labels("test_ds", {0: "negative_class", 1: "positive_class"})
+        kls.tokenizer.fit(["o gato dorme", "o cachorro corre", "teste de predição"])
+        for _ in range(3):
+            kls.add_data(
+                ["o gato dorme", "o cachorro corre", "a menina brinca", "o pássaro voa"],
+                [0, 1, 0, 1],
+                label_strings={0: "negative_class", 1: "positive_class"},
+                dataset_name="test_ds",
+            )
+        kls.check_training_start()
+        kls.train_som_on_buffer()
+        kls._label_neurons()
+        pred = kls.predict("o gato dorme", dataset_name="test_ds")
+        assert pred in ["negative_class", "positive_class"], \
+            f"predict returned: {pred} (expected dynamic label)"
+        assert pred != "gato" and pred != "cachorro", \
+            f"predict returned hardcoded: {pred}"
+        checks["predict_dynamic_labels"] = {
+            "status": "PASS",
+            "details": f"predict returned: {pred} (dynamic, not hardcoded)",
+        }
+    except Exception as e:
+        checks["predict_dynamic_labels"] = {"status": "FAIL", "error": str(e)}
+
+    # Check 3: process_batch_v2 returns phase info
+    try:
+        if kls is None:
+            raise RuntimeError("KLS not instantiated")
+        result = kls.process_batch_v2(
+            ["o gato dorme", "o cachorro corre"],
+            [0, 1],
+            dataset_name="test_ds",
+            enable_punishment=False,
+        )
+        assert result["phase"] == "CONHECIMENTO"
+        assert "elapsed_ms" in result
+        checks["process_batch_v2_conhecimento"] = {
+            "status": "PASS",
+            "details": f"phase={result['phase']}, action={result['action']}",
+        }
+    except Exception as e:
+        checks["process_batch_v2_conhecimento"] = {"status": "FAIL", "error": str(e)}
+
+    # Check 4: V2 methods exist
+    try:
+        for method in ["train_hypotheses", "select_best_delta",
+                       "apply_best_delta_and_consolidate", "process_batch_v2",
+                       "get_v2_metrics"]:
+            assert hasattr(kls, method), f"missing method: {method}"
+        checks["v2_methods_exist"] = {
+            "status": "PASS",
+            "details": "train_hypotheses, select_best_delta, apply_best_delta_and_consolidate, "
+                       "process_batch_v2, get_v2_metrics",
+        }
+    except Exception as e:
+        checks["v2_methods_exist"] = {"status": "FAIL", "error": str(e)}
+
+    # Summary
+    n_pass = sum(1 for c in checks.values() if c.get("status") == "PASS")
+    n_fail = sum(1 for c in checks.values() if c.get("status") == "FAIL")
+    return {
+        "checks": checks,
+        "n_pass": n_pass,
+        "n_fail": n_fail,
+        "all_pass": n_fail == 0,
+    }
+
+
+# ============================================================================
+# 9. PHASE 1 — CONHECIMENTO (streaming 8 datasets, sem punição)
+# ============================================================================
+def run_fase_conhecimento(
+    kls: KohonenLearningSystemV2,
+    hf_token: Optional[str],
+) -> Dict[str, Any]:
+    """Fase 1 — CONHECIMENTO: streaming 8 datasets, sem punição.
+
+    User requirement: "streaming CONHECIMENTO de 100 em 100 samples nesta
+    sequência [8 datasets]" + "meta mínima 8000 samples ou mais para
+    CONHECIMENTO"
+
+    Para cada dataset:
+      - Stream em chunks de 100 samples.
+      - process_batch_v2(enable_punishment=False) — apenas acumula conhecimento.
+      - Pausas inter-batch e inter-dataset.
+
+    Returns:
+        Dict com métricas da fase CONHECIMENTO.
+    """
+    logger.info("\n" + "=" * 80)
+    logger.info("[V6.5-V2] FASE 1 — CONHECIMENTO (sem punição)")
+    logger.info("=" * 80)
+    logger.info(f"  Datasets: {len(CONHECIMENTO_DATASETS)}")
+    logger.info(f"  Max samples/dataset: {MAX_SAMPLES_PER_DATASET_CONHECIMENTO}")
+    logger.info(f"  Meta mínima: {META_MINIMA_CONHECIMENTO}")
+    logger.info(f"  Stream batch size: {STREAM_BATCH_SIZE}")
+    logger.info("=" * 80 + "\n")
+
+    step = 0
+    t_start = time.time()
+    samples_per_dataset: Dict[str, int] = {ds: 0 for ds in CONHECIMENTO_DATASETS}
+    streaming_failures: Dict[str, int] = {ds: 0 for ds in CONHECIMENTO_DATASETS}
+    storage_critical_stopped = False
+    chunk_idx_global = 0
+    accuracies_log: List[Dict[str, Any]] = []
+    attention_log: List[Dict[str, Any]] = []
+
+    for ds_idx, dataset_name in enumerate(CONHECIMENTO_DATASETS):
+        if storage_critical_stopped:
+            break
+        total_so_far = sum(samples_per_dataset.values())
+        if total_so_far >= META_MINIMA_CONHECIMENTO:
+            logger.info(
+                f"[V6.5-V2] Meta CONHECIMENTO atingida ({total_so_far} ≥ "
+                f"{META_MINIMA_CONHECIMENTO}). Parando Fase 1."
+            )
+            break
+        logger.info(
+            f"\n[V6.5-V2] {'='*40} DATASET {ds_idx+1}/{len(CONHECIMENTO_DATASETS)}: "
+            f"{dataset_name} {'='*40}"
+        )
+        logger.info(
+            f"[V6.5-V2] Streaming {dataset_name} in chunks of {STREAM_BATCH_SIZE}..."
+        )
+
+        try:
+            chunk_iter = stream_dataset_in_chunks(
+                dataset_name=dataset_name,
+                chunk_size=STREAM_BATCH_SIZE,
+                max_total=MAX_SAMPLES_PER_DATASET_CONHECIMENTO,
+                hf_token=hf_token,
+                timeout_per_chunk=90,
+            )
+            for chunk_samples in chunk_iter:
+                if storage_critical_stopped:
+                    break
+                total_so_far = sum(samples_per_dataset.values())
+                if total_so_far >= META_MINIMA_CONHECIMENTO:
+                    logger.info(
+                        f"[V6.5-V2] Meta CONHECIMENTO atingida durante {dataset_name}."
+                    )
+                    break
+                chunk_idx_global += 1
+                if not chunk_samples:
+                    streaming_failures[dataset_name] += 1
+                    continue
+                samples_per_dataset[dataset_name] += len(chunk_samples)
+                labels = [make_label(s, dataset_name) for s in chunk_samples]
+                logger.info(
+                    f"[V6.5-V2] Processing chunk {chunk_idx_global} "
+                    f"({len(chunk_samples)} samples) from {dataset_name}"
+                )
+
+                # Processa em batches COM PAUSAS
+                for batch_start in range(0, len(chunk_samples), BATCH_SIZE):
+                    try:
+                        if step > 0 and step % STORAGE_CHECK_INTERVAL_STEPS == 0:
+                            disk_pct = get_disk_usage_pct()
+                            rss_mb = get_process_rss_mb()
+                            cg_limit = get_cgroup_memory_limit_mb()
+                            if cg_limit is not None:
+                                mem_pct = (rss_mb / cg_limit) * 100.0
+                                logger.info(
+                                    f"[V6.5-V2] MEM: RSS={rss_mb:.0f}MB / "
+                                    f"cg_limit={cg_limit:.0f}MB ({mem_pct:.1f}%)"
+                                )
+                                if mem_pct > CGROUP_MEM_CRITICAL_PCT:
+                                    logger.error(
+                                        f"[V6.5-V2] MEM CRITICAL: RSS={rss_mb:.0f}MB "
+                                        f"({mem_pct:.1f}% of {cg_limit:.0f}MB). "
+                                        f"STOPPING CONHECIMENTO (graceful)."
+                                    )
+                                    total_now = sum(samples_per_dataset.values())
+                                    save_model_states_for_evaluation(
+                                        kls=kls,
+                                        reason=f"mem_critical_conhecimento_{mem_pct:.1f}pct",
+                                        step=step,
+                                        total_samples=total_now,
+                                    )
+                                    storage_critical_stopped = True
+                                    break
+                            if disk_pct > STORAGE_CRITICAL_PCT:
+                                logger.error(
+                                    f"[V6.5-V2] STORAGE CRITICAL: disk={disk_pct:.1f}% "
+                                    f"> threshold={STORAGE_CRITICAL_PCT}%. "
+                                    f"STOPPING CONHECIMENTO."
+                                )
+                                total_now = sum(samples_per_dataset.values())
+                                save_model_states_for_evaluation(
+                                    kls=kls,
+                                    reason=f"storage_critical_conhecimento_disk_{disk_pct:.1f}pct",
+                                    step=step,
+                                    total_samples=total_now,
+                                )
+                                storage_critical_stopped = True
+                                break
+
+                        batch_sents = chunk_samples[batch_start: batch_start + BATCH_SIZE]
+                        batch_labels = labels[batch_start: batch_start + BATCH_SIZE]
+                        try:
+                            # FASE 1: enable_punishment=False (apenas CONHECIMENTO)
+                            result = kls.process_batch_v2(
+                                batch_sents,
+                                batch_labels,
+                                label_strings=DATASET_LABEL_STRINGS.get(dataset_name),
+                                dataset_name=dataset_name,
+                                enable_punishment=False,
+                            )
+                        except Exception as e:
+                            logger.error(f"[V6.5-V2] process_batch_v2 error: {e}")
+                            traceback.print_exc()
+                            continue
+
+                        # Métricas
+                        acc = kls.evaluate_classification()
+                        accuracies_log.append({
+                            "step": step,
+                            "dataset": dataset_name,
+                            "chunk_idx": chunk_idx_global,
+                            "batch_size": len(batch_sents),
+                            "accuracy": float(acc),
+                            "phase": "CONHECIMENTO",
+                            "buffer_size": int(len(kls.buffer_4d)),
+                            "training_ready": bool(kls.training_ready),
+                            "elapsed_ms": float(result.get("elapsed_ms", 0.0)),
+                        })
+
+                        # Attention log
+                        attn_metrics = kls.get_attention_metrics()
+                        if attn_metrics.get("n_calls", 0) > len(attention_log) * 16:
+                            attention_log.append({
+                                "step": step,
+                                "dataset": dataset_name,
+                                "n_calls": int(attn_metrics.get("n_calls", 0)),
+                                "n_errors": int(attn_metrics.get("n_errors", 0)),
+                                "last_attn_activated": bool(attn_metrics.get("last_attn_activated", False)),
+                                "logic_functional": bool(attn_metrics.get("logic_functional", False)),
+                            })
+
+                        step += 1
+                        time.sleep(INTER_BATCH_PAUSE_S)
+                    except Exception as e:
+                        logger.error(f"[V6.5-V2] Batch error: {e}")
+                        traceback.print_exc()
+                        continue
+
+                time.sleep(INTER_STREAM_BATCH_PAUSE_S)
+                # V2-dynamic-memory — Buffer sliding window: trunca para os
+                # últimos MAX_BUFFER_SIZE amostras após cada chunk.
+                # Os pesos do SOM já capturam o conhecimento acumulado,
+                # então o buffer só precisa conter amostras recentes.
+                if len(kls.buffer_4d) > MAX_BUFFER_SIZE:
+                    overflow = len(kls.buffer_4d) - MAX_BUFFER_SIZE
+                    del kls.buffer_4d[:overflow]
+                    del kls.buffer_labels[:overflow]
+                    logger.info(
+                        f"[V6.5-V2] Buffer trimmed: -{overflow} → "
+                        f"{len(kls.buffer_4d)} samples (cap={MAX_BUFFER_SIZE})"
+                    )
+                # Aggressive memory cleanup between chunks
+                if chunk_idx_global % 2 == 0:
+                    aggressive_memory_cleanup()
+        except Exception as e:
+            logger.error(f"[V6.5-V2] Dataset {dataset_name} failed: {e}")
+            traceback.print_exc()
+            streaming_failures[dataset_name] += 1
+
+        time.sleep(INTER_DATASET_PAUSE_S)
+
+    t_elapsed = time.time() - t_start
+    total_samples = sum(samples_per_dataset.values())
+
+    logger.info(f"\n[V6.5-V2] FASE 1 (CONHECIMENTO) concluída:")
+    logger.info(f"  Total samples: {total_samples}")
+    logger.info(f"  Meta atingida: {total_samples >= META_MINIMA_CONHECIMENTO}")
+    logger.info(f"  Tempo total: {t_elapsed:.1f}s")
+    logger.info(f"  Storage critical stopped: {storage_critical_stopped}")
+
+    return {
+        "phase": "CONHECIMENTO",
+        "total_samples": int(total_samples),
+        "meta_minima": int(META_MINIMA_CONHECIMENTO),
+        "meta_atingida": bool(total_samples >= META_MINIMA_CONHECIMENTO),
+        "samples_per_dataset": samples_per_dataset,
+        "streaming_failures": streaming_failures,
+        "storage_critical_stopped": bool(storage_critical_stopped),
+        "elapsed_s": float(t_elapsed),
+        "n_steps": int(step),
+        "n_chunks": int(chunk_idx_global),
+        "accuracies_log": accuracies_log[-50:],  # últimos 50
+        "attention_log": attention_log[-20:],
+        "final_state": kls.get_v2_metrics(),
+    }
+
+
+# ============================================================================
+# 10. PHASE 2 — TREINAMENTO COM PUNIÇÃO (16 hipóteses × 3 tentativas)
+# ============================================================================
+def run_fase_punicão(
+    kls: KohonenLearningSystemV2,
+    hf_token: Optional[str],
+) -> Dict[str, Any]:
+    """Fase 2 — TREINAMENTO COM PUNIÇÃO: streaming 500 samples + V2 protocol.
+
+    User requirement: "dataset para TREINAMENTO COM PUNIÇÃO ATIVA
+    'BrunoN-Dev/corpus-ptbr-v1'" + "meta mínima 500 para TREINAMENTO COM
+    PUNIÇÃO" + "Cada evento de erro (punição) dispara 3 tentativas de ajuste.
+    Em cada tentativa, 16 geradores paralelos de hipóteses produzem candidatos
+    a delta. O melhor delta é aplicado ao MAPA-SOM. Após a aplicação,
+    consolidação via EWC."
+
+    Returns:
+        Dict com métricas da fase TREINAMENTO COM PUNIÇÃO.
+    """
+    logger.info("\n" + "=" * 80)
+    logger.info("[V6.5-V2] FASE 2 — TREINAMENTO COM PUNIÇÃO (16 hipóteses × 3 tentativas)")
+    logger.info("=" * 80)
+    logger.info(f"  Dataset: {PUNICAO_DATASET}")
+    logger.info(f"  Max samples: {MAX_SAMPLES_PUNICAO}")
+    logger.info(f"  Meta mínima: {META_MINIMA_PUNICAO}")
+    logger.info(f"  n_hypotheses: {N_HYPOTHESES}")
+    logger.info(f"  n_trials: {N_TRIALS}")
+    logger.info(f"  hyp_train_steps: {HYP_TRAIN_STEPS}")
+    logger.info("=" * 80 + "\n")
+
+    step = 0
+    t_start = time.time()
+    samples_processed = 0
+    storage_critical_stopped = False
+    chunk_idx_global = 0
+    punishment_log: List[Dict[str, Any]] = []
+    hypotheses_log: List[Dict[str, Any]] = []
+    accuracies_log: List[Dict[str, Any]] = []
+    delta_applications_log: List[Dict[str, Any]] = []
+
+    try:
+        chunk_iter = stream_dataset_in_chunks(
+            dataset_name=PUNICAO_DATASET,
+            chunk_size=STREAM_BATCH_SIZE,
+            max_total=MAX_SAMPLES_PUNICAO,
+            hf_token=hf_token,
+            timeout_per_chunk=90,
+        )
+        for chunk_samples in chunk_iter:
+            if storage_critical_stopped:
+                break
+            if samples_processed >= META_MINIMA_PUNICAO:
+                logger.info(
+                    f"[V6.5-V2] Meta PUNIÇÃO atingida ({samples_processed} ≥ "
+                    f"{META_MINIMA_PUNICAO})."
+                )
+                break
+            chunk_idx_global += 1
+            if not chunk_samples:
+                continue
+            samples_processed += len(chunk_samples)
+            labels = [make_label(s, PUNICAO_DATASET) for s in chunk_samples]
+            logger.info(
+                f"[V6.5-V2] PUNIÇÃO chunk {chunk_idx_global} "
+                f"({len(chunk_samples)} samples) from {PUNICAO_DATASET}"
+            )
+
+            for batch_start in range(0, len(chunk_samples), BATCH_SIZE):
+                try:
+                    if step > 0 and step % STORAGE_CHECK_INTERVAL_STEPS == 0:
+                        disk_pct = get_disk_usage_pct()
+                        if disk_pct > STORAGE_CRITICAL_PCT:
+                            logger.error(
+                                f"[V6.5-V2] STORAGE CRITICAL: disk={disk_pct:.1f}%. "
+                                f"STOPPING PUNIÇÃO."
+                            )
+                            save_model_states_for_evaluation(
+                                kls=kls,
+                                reason=f"storage_critical_punicão_disk_{disk_pct:.1f}pct",
+                                step=step,
+                                total_samples=samples_processed,
+                            )
+                            storage_critical_stopped = True
+                            break
+
+                    batch_sents = chunk_samples[batch_start: batch_start + BATCH_SIZE]
+                    batch_labels = labels[batch_start: batch_start + BATCH_SIZE]
+                    try:
+                        # FASE 2: enable_punishment=True (V2 protocol)
+                        result = kls.process_batch_v2(
+                            batch_sents,
+                            batch_labels,
+                            label_strings=DATASET_LABEL_STRINGS.get(PUNICAO_DATASET),
+                            dataset_name=PUNICAO_DATASET,
+                            enable_punishment=True,
+                        )
+                    except Exception as e:
+                        logger.error(f"[V6.5-V2] process_batch_v2 error: {e}")
+                        traceback.print_exc()
+                        continue
+
+                    # Log punishment events
+                    action = result.get("action", "none")
+                    if action != "none" and action != "success":
+                        punishment_log.append({
+                            "step": step,
+                            "action": action,
+                            "accuracy": float(result.get("accuracy", 0.0)),
+                            "punishment_count": int(kls.punishment_count),
+                            "cycle_completed": bool(result.get("cycle_completed", False)),
+                        })
+                        # Log hypotheses training
+                        if "hypotheses_training" in result:
+                            hyp_info = result["hypotheses_training"]
+                            hypotheses_log.append({
+                                "step": step,
+                                "loss_initial": float(hyp_info.get("loss_initial", 0.0)),
+                                "loss_final": float(hyp_info.get("loss_final", 0.0)),
+                                "delta_scale": float(hyp_info.get("delta_scale_final", 0.0)),
+                                "elapsed_ms": float(hyp_info.get("elapsed_ms", 0.0)),
+                            })
+                        # Log delta applications
+                        if "delta_application" in result:
+                            apply_info = result["delta_application"]
+                            delta_applications_log.append({
+                                "step": step,
+                                "acc_before": float(apply_info.get("acc_before", 0.0)),
+                                "acc_after": float(apply_info.get("acc_after", 0.0)),
+                                "best_acc_during_selection": float(apply_info.get("best_acc_during_selection", 0.0)),
+                                "delta_norm": float(apply_info.get("delta_norm", 0.0)),
+                                "ewc_reference_set": bool(apply_info.get("ewc_reference_set", False)),
+                                "elapsed_ms": float(apply_info.get("elapsed_ms", 0.0)),
+                            })
+
+                    # Accuracy log
+                    acc = kls.evaluate_classification()
+                    accuracies_log.append({
+                        "step": step,
+                        "accuracy": float(acc),
+                        "phase": "PUNICAO",
+                        "action": action,
+                        "buffer_size": int(len(kls.buffer_4d)),
+                        "punishment_count": int(kls.punishment_count),
+                        "success_count": int(kls.success_count),
+                        "ewc_reference_set": bool(kls.som.old_weights_w is not None),
+                    })
+
+                    step += 1
+                    time.sleep(INTER_BATCH_PAUSE_S)
+                except Exception as e:
+                    logger.error(f"[V6.5-V2] Batch error in PUNIÇÃO: {e}")
+                    traceback.print_exc()
+                    continue
+
+            time.sleep(INTER_STREAM_BATCH_PAUSE_S)
+            # V2-dynamic-memory — Buffer sliding window na PUNIÇÃO também
+            if len(kls.buffer_4d) > MAX_BUFFER_SIZE:
+                overflow = len(kls.buffer_4d) - MAX_BUFFER_SIZE
+                del kls.buffer_4d[:overflow]
+                del kls.buffer_labels[:overflow]
+                logger.info(
+                    f"[V6.5-V2] PUNIÇÃO buffer trimmed: -{overflow} → "
+                    f"{len(kls.buffer_4d)} samples"
+                )
+            if chunk_idx_global % 2 == 0:
+                aggressive_memory_cleanup()
+    except Exception as e:
+        logger.error(f"[V6.5-V2] PUNIÇÃO dataset failed: {e}")
+        traceback.print_exc()
+
+    t_elapsed = time.time() - t_start
+
+    logger.info(f"\n[V6.5-V2] FASE 2 (PUNIÇÃO) concluída:")
+    logger.info(f"  Total samples: {samples_processed}")
+    logger.info(f"  Meta atingida: {samples_processed >= META_MINIMA_PUNICAO}")
+    logger.info(f"  Tempo total: {t_elapsed:.1f}s")
+    logger.info(f"  Punishment events: {len(punishment_log)}")
+    logger.info(f"  Hypotheses trainings: {len(hypotheses_log)}")
+    logger.info(f"  Delta applications: {len(delta_applications_log)}")
+
+    return {
+        "phase": "TREINAMENTO_COM_PUNICAO",
+        "dataset": PUNICAO_DATASET,
+        "total_samples": int(samples_processed),
+        "meta_minima": int(META_MINIMA_PUNICAO),
+        "meta_atingida": bool(samples_processed >= META_MINIMA_PUNICAO),
+        "storage_critical_stopped": bool(storage_critical_stopped),
+        "elapsed_s": float(t_elapsed),
+        "n_steps": int(step),
+        "n_chunks": int(chunk_idx_global),
+        "punishment_events": punishment_log,
+        "hypotheses_trainings": hypotheses_log,
+        "delta_applications": delta_applications_log,
+        "accuracies_log": accuracies_log[-50:],
+        "final_v2_state": kls.get_v2_metrics(),
+    }
+
+
+# ============================================================================
+# 11. Evaluate attention (ativo e logicamente funcional)
+# ============================================================================
+def evaluate_attention(kls: KohonenLearningSystemV2) -> Dict[str, Any]:
+    """V6.5-V2 — Avalia se o mecanismo de atenção está ativo e funcional."""
+    metrics_snapshot = kls.get_attention_metrics()
+    return {
+        "evaluation": "attention_v65_v2",
+        "user_requirement": "verificar se o mecanismo de atenção está ativo e acessado logicamente funcional",
+        "metrics": metrics_snapshot,
+        "active": bool(metrics_snapshot.get("active", False)),
+        "logic_functional": bool(metrics_snapshot.get("logic_functional", False)),
+        "n_calls": int(metrics_snapshot.get("n_calls", 0)),
+        "n_errors": int(metrics_snapshot.get("n_errors", 0)),
+        "n_heads": int(metrics_snapshot.get("n_heads", 0)),
+        "assessment": "PASS" if metrics_snapshot.get("logic_functional") else "FAIL",
+    }
+
+
+# ============================================================================
+# 12. Evaluate predict fix (rótulos dinâmicos, não hardcoded)
+# ============================================================================
+def evaluate_predict_fix(kls: KohonenLearningSystemV2) -> Dict[str, Any]:
+    """V6.5-V2 — Avalia que predict() NÃO retorna mais 'gato'/'cachorro' fixos."""
+    test_queries = [
+        "o gato dorme na cama",
+        "calcule dois mais dois",
+        "qual é a capital do brasil",
+        "explique o que é uma rede neural",
+        "olá como você está",
+        "traduza hello para portugues",
+    ]
+    results = []
+    n_returns_gato = 0
+    n_returns_cachorro = 0
+    n_returns_registry_label = 0
+    n_returns_default_label = 0
+
+    registry_state = kls.get_label_registry_state()
+    all_registered_labels = set()
+    for ds_name, mapping in registry_state["datasets"].items():
+        for label_str in mapping.values():
+            all_registered_labels.add(label_str)
+    default_labels = set(registry_state["default_label_strings"].values())
+
+    for q in test_queries:
+        try:
+            pred = kls.predict(q)
+        except Exception as e:
+            pred = f"<error: {e}>"
+        try:
+            pred_proba, prob = kls.predict_proba(q)
+        except Exception as e:
+            pred_proba, prob = f"<error: {e}>", -1.0
+
+        is_gato = (pred == "gato")
+        is_cachorro = (pred == "cachorro")
+        is_registry = pred in all_registered_labels
+        is_default = pred in default_labels
+
+        if is_gato: n_returns_gato += 1
+        if is_cachorro: n_returns_cachorro += 1
+        if is_registry: n_returns_registry_label += 1
+        if is_default: n_returns_default_label += 1
+
+        results.append({
+            "query": q,
+            "prediction": str(pred),
+            "probability": float(prob) if isinstance(prob, (int, float)) else -1.0,
+            "is_gato_hardcoded": bool(is_gato),
+            "is_cachorro_hardcoded": bool(is_cachorro),
+            "is_registry_label": bool(is_registry),
+            "is_default_label": bool(is_default),
+        })
+
+    # Per-dataset check
+    per_dataset_results = []
+    for ds_name, mapping in registry_state["datasets"].items():
+        try:
+            pred = kls.predict("teste de predição", dataset_name=ds_name)
+            valid = pred in set(mapping.values())
+        except Exception as e:
+            pred = f"<error: {e}>"
+            valid = False
+        per_dataset_results.append({
+            "dataset": ds_name,
+            "expected_labels": list(mapping.values()),
+            "prediction": str(pred),
+            "valid_for_dataset": bool(valid),
+        })
+
+    return {
+        "evaluation": "predict_fix_v65_v2",
+        "user_requirement": "os strings 'gato' e 'cachorro' são fixos quando deveriam ser extrações variáveis e flexíveis de rótulos proveniente de dados dos datasets anteriormente treinados",
+        "n_test_queries": len(test_queries),
+        "results": results,
+        "per_dataset_results": per_dataset_results,
+        "summary": {
+            "n_returns_gato": int(n_returns_gato),
+            "n_returns_cachorro": int(n_returns_cachorro),
+            "n_returns_registry_label": int(n_returns_registry_label),
+            "n_returns_default_label": int(n_returns_default_label),
+            "hardcoded_bug_present": bool(n_returns_gato > 0 or n_returns_cachorro > 0),
+            "predict_fix_verified": bool(n_returns_gato == 0 and n_returns_cachorro == 0),
+        },
+        "quality_assessment": {
+            "fix_status": "PASS" if (n_returns_gato == 0 and n_returns_cachorro == 0) else "FAIL",
+            "note": "predict() não retorna mais 'gato'/'cachorro' fixos — usa label_registry dinâmico.",
+        },
+    }
+
+
+# ============================================================================
+# 13. Launch user questions WITHOUT help (3 perguntas, sem auxílio)
+# ============================================================================
+def launch_user_questions_without_help(
+    kls: KohonenLearningSystemV2,
+) -> Dict[str, Any]:
+    """V6.5-V2 — Envia as 3 perguntas específicas SEM nenhum contexto.
+
+    User requirement: "não ajudar o modelo em respostas e lançar perguntas:
+    'Luva de Pedreiro Távila' e 'Lula reserva valor' e
+    'Amazonas força-tarefa vítimas'"
+
+    IMPORTANTE: As perguntas são enviadas EXATAMENTE como o usuário as escreveu,
+    sem nenhum prompt adicional, sem contexto, sem instrução de formato,
+    sem system prompt, sem few-shot examples.
+    """
+    user_questions = [
+        "Luva de Pedreiro Távila",
+        "Lula reserva valor",
+        "Amazonas força-tarefa vítimas",
+    ]
+    results = []
+    for q in user_questions:
+        t_start = time.time()
+        try:
+            som_pred = kls.predict(q)
+        except Exception as e:
+            som_pred = f"<predict_error: {e}>"
+        try:
+            reasoning_text = kls.reason_sync(q)
+        except Exception as e:
+            reasoning_text = f"<reasoning_error: {e}>"
+        latency_ms = (time.time() - t_start) * 1000.0
+        think_match = re.search(r"<think>(.*?)</think>", reasoning_text, re.DOTALL)
+        plan_match = re.search(r"<plan>(.*?)</plan>", reasoning_text, re.DOTALL)
+        answer_match = re.search(r"<answer>(.*?)</answer>", reasoning_text, re.DOTALL)
+        decompose_match = re.search(r"<decompose>(.*?)</decompose>", reasoning_text, re.DOTALL)
+        results.append({
+            "query": q,
+            "query_was_modified": False,
+            "context_provided": False,
+            "system_prompt_used": False,
+            "few_shot_examples": False,
+            "som_prediction": str(som_pred),
+            "reasoning_length": int(len(reasoning_text)),
+            "has_think": bool(think_match),
+            "has_plan": bool(plan_match),
+            "has_answer": bool(answer_match),
+            "has_decompose": bool(decompose_match),
+            "think_preview": (
+                think_match.group(1).strip()[:200] + ("..." if len(think_match.group(1)) > 200 else "")
+            ) if think_match else "",
+            "answer_preview": (
+                answer_match.group(1).strip()[:200] + ("..." if len(answer_match.group(1)) > 200 else "")
+            ) if answer_match else "",
+            "raw_response_preview": reasoning_text[:500] + ("..." if len(reasoning_text) > 500 else ""),
+            "n_tags": sum(1 for tag in ["<think>", "<plan>", "<decompose>", "<answer>"] if tag in reasoning_text),
+            "latency_ms": float(latency_ms),
+        })
+    n_answer = sum(1 for r in results if r["has_answer"])
+    n_think = sum(1 for r in results if r["has_think"])
+    return {
+        "evaluation": "user_questions_without_help_v65_v2",
+        "user_requirement": "não ajudar o modelo em respostas e lançar perguntas",
+        "questions_sent_verbatim": True,
+        "no_context_added": True,
+        "no_system_prompt": True,
+        "no_few_shot": True,
+        "n_questions": len(user_questions),
+        "questions": user_questions,
+        "results": results,
+        "summary": {
+            "n_with_answer": n_answer,
+            "n_with_think": n_think,
+            "answer_rate": float(n_answer / max(1, len(results))),
+            "think_rate": float(n_think / max(1, len(results))),
+            "avg_latency_ms": float(sum(r["latency_ms"] for r in results) / max(1, len(results))),
+            "avg_reasoning_length": float(sum(r["reasoning_length"] for r in results) / max(1, len(results))),
+        },
+        "quality_assessment": {
+            "model_not_helped": True,
+            "note": "As 3 perguntas foram enviadas verbatim, sem system prompt, sem few-shot, sem contexto adicional.",
+        },
+    }
+
+
+# ============================================================================
+# 13.5 HF Batch Upload — envia scripts+estado+modelo ao HF em lote
+# ============================================================================
+HF_REPO_ID = "PowerMachine/BiGRU_T_version"
+
+
+def upload_to_hf_batch(
+    hf_token: str,
+    files_to_upload: List[Path],
+    repo_id: str = HF_REPO_ID,
+    repo_type: str = "model",
+) -> Dict[str, Any]:
+    """V6.5-V2-dynamic — Envia arquivos ao HF em LOTE (commit único).
+
+    User requirement: "atualizações para o HF devem ser feitas em lote" +
+    "após concluir enviar scripts e arquivos e estado do modelo testados e
+    aprovados sobrescrevendo os antigos desatualizados".
+
+    Args:
+        hf_token: token HF (será apagado após uso).
+        files_to_upload: lista de paths locais a enviar.
+        repo_id: repositório HF destino.
+        repo_type: 'model', 'dataset' ou 'space'.
+
+    Returns:
+        Dict com status de cada upload e URL do commit.
+    """
+    if not hf_token:
+        return {"uploaded": False, "reason": "no_hf_token"}
+
+    try:
+        from huggingface_hub import HfApi, whoami
+    except ImportError as e:
+        logger.error(f"[V6.5-V2] huggingface_hub not installed: {e}")
+        return {"uploaded": False, "reason": f"import_error: {e}"}
+
+    api = HfApi(token=hf_token)
+    try:
+        user_info = whoami(token=hf_token)
+        logger.info(f"[V6.5-V2] HF authenticated as: {user_info.get('name', 'unknown')}")
+    except Exception as e:
+        logger.error(f"[V6.5-V2] HF auth failed: {e}")
+        return {"uploaded": False, "reason": f"auth_failed: {e}"}
+
+    # Garante que o repo existe
+    try:
+        api.repo_info(repo_id=repo_id, repo_type=repo_type)
+    except Exception:
+        try:
+            api.create_repo(repo_id=repo_id, repo_type=repo_type, exist_ok=True)
+            logger.info(f"[V6.5-V2] Created HF repo: {repo_id}")
+        except Exception as e:
+            logger.warning(f"[V6.5-V2] Could not create repo {repo_id}: {e}")
+
+    # Filtra arquivos existentes
+    upload_plan: List[Dict[str, Any]] = []
+    for src in files_to_upload:
+        if not src.exists():
+            logger.warning(f"[V6.5-V2] Skipping non-existent file: {src}")
+            continue
+        # Caminho no repo: mantém apenas o nome do arquivo (sem dirs locais)
+        path_in_repo = src.name
+        upload_plan.append({
+            "local": str(src),
+            "path_in_repo": path_in_repo,
+            "size_mb": float(src.stat().st_size / 1e6),
+        })
+
+    if not upload_plan:
+        return {"uploaded": False, "reason": "no_files_to_upload"}
+
+    logger.info(
+        f"[V6.5-V2] Uploading {len(upload_plan)} files to HF "
+        f"repo={repo_id} in BATCH (single commit):"
+    )
+    for item in upload_plan:
+        logger.info(f"  - {item['local']} → {item['path_in_repo']} "
+                    f"({item['size_mb']:.2f}MB)")
+
+    # Upload em lote (commit único): usa upload_folder com todos os arquivos
+    # numa pasta temporária — ou, mais simples, upload_file individual mas
+    # em sequência rápida. Para commit único verdadeiro, usaremos create_commit.
+    results: List[Dict[str, Any]] = []
+    try:
+        # Método 1: create_commit com múltiplos operations (commit único)
+        from huggingface_hub import CommitOperationAdd
+        operations = []
+        for item in upload_plan:
+            operations.append(CommitOperationAdd(
+                path_in_repo=item["path_in_repo"],
+                path_or_fileobj=item["local"],
+            ))
+        commit_info = api.create_commit(
+            repo_id=repo_id,
+            repo_type=repo_type,
+            operations=operations,
+            commit_message=(
+                f"V6.5-V2-dynamic: upload batch (scripts + state + model) "
+                f"[{len(operations)} files]"
+            ),
+        )
+        logger.info(f"[V6.5-V2] Batch commit created: {commit_info}")
+        for item in upload_plan:
+            results.append({
+                "file": item["path_in_repo"],
+                "uploaded": True,
+                "size_mb": item["size_mb"],
+            })
+        return {
+            "uploaded": True,
+            "n_files": len(results),
+            "commit_url": str(commit_info.commit_url) if hasattr(commit_info, "commit_url") else str(commit_info),
+            "results": results,
+        }
+    except Exception as e:
+        logger.error(f"[V6.5-V2] Batch commit failed: {e}")
+        traceback.print_exc()
+
+    # Método 2 (fallback): upload individual
+    logger.info("[V6.5-V2] Falling back to individual uploads...")
+    for item in upload_plan:
+        try:
+            api.upload_file(
+                path_or_fileobj=item["local"],
+                path_in_repo=item["path_in_repo"],
+                repo_id=repo_id,
+                repo_type=repo_type,
+            )
+            results.append({
+                "file": item["path_in_repo"],
+                "uploaded": True,
+                "size_mb": item["size_mb"],
+            })
+            logger.info(f"  ✓ {item['path_in_repo']} uploaded")
+        except Exception as e:
+            results.append({
+                "file": item["path_in_repo"],
+                "uploaded": False,
+                "error": str(e)[:200],
+            })
+            logger.error(f"  ✗ {item['path_in_repo']} failed: {e}")
+
+    return {
+        "uploaded": any(r.get("uploaded") for r in results),
+        "n_files": len(results),
+        "n_uploaded": sum(1 for r in results if r.get("uploaded")),
+        "results": results,
+    }
+
+
+def scrub_hf_token_from_scripts(scripts_dir: Path) -> Dict[str, Any]:
+    """V6.5-V2-dynamic — Apaga HF_TOKEN de todos os scripts no diretório.
+
+    User requirement: "HF_TOKEN deve ser apagada dos scripts enviados ao HF
+    após uso".
+
+    Usa regex para detectar qualquer string que pareça um token HF
+    (hf_<alphanumeric>{32,}) e substitui por '<HF_TOKEN_REMOVED>'.
+    """
+    pattern = re.compile(r'hf_[A-Za-z0-9]{32,}')
+    n_files_scanned = 0
+    n_files_modified = 0
+    n_tokens_removed = 0
+    for py_file in scripts_dir.rglob("*.py"):
+        n_files_scanned += 1
+        try:
+            content = py_file.read_text(encoding="utf-8")
+            matches = pattern.findall(content)
+            if matches:
+                new_content = pattern.sub("<HF_TOKEN_REMOVED>", content)
+                py_file.write_text(new_content, encoding="utf-8")
+                n_files_modified += 1
+                n_tokens_removed += len(matches)
+                logger.info(f"[V6.5-V2] Scrubbed {len(matches)} HF token(s) from {py_file}")
+        except Exception as e:
+            logger.warning(f"[V6.5-V2] Could not scan {py_file}: {e}")
+    return {
+        "n_files_scanned": n_files_scanned,
+        "n_files_modified": n_files_modified,
+        "n_tokens_removed": n_tokens_removed,
+    }
+
+
+
+def main() -> int:
+    import torch
+
+    n_neurons = SOM_GRID[0] * SOM_GRID[1] * SOM_GRID[2] * SOM_GRID[3]
+    print("\n" + "=" * 80)
+    print("V6.5-V2 — 16 HIPÓTESES + 3 TENTATIVAS + 2 FASES (CONHECIMENTO → PUNIÇÃO)")
+    print("=" * 80)
+    print(f"  BATCH_SIZE           : {BATCH_SIZE}")
+    print(f"  STREAM_BATCH_SIZE    : {STREAM_BATCH_SIZE}  (user: 'de 100 em 100')")
+    print(f"  CONHECIMENTO datasets: {len(CONHECIMENTO_DATASETS)}")
+    print(f"  PUNICAO dataset      : {PUNICAO_DATASET}")
+    print(f"  Max samples CONHEC   : {MAX_SAMPLES_PER_DATASET_CONHECIMENTO}/dataset")
+    print(f"  Max samples PUNICAO  : {MAX_SAMPLES_PUNICAO}")
+    print(f"  Meta CONHECIMENTO    : ≥{META_MINIMA_CONHECIMENTO}")
+    print(f"  Meta PUNICAO         : ≥{META_MINIMA_PUNICAO}")
+    print(f"  SOM grid             : {SOM_GRID} ({n_neurons} neurons)")
+    print(f"  HIDDEN_DIM           : {HIDDEN_DIM}")
+    print(f"  VOCAB_SIZE           : {VOCAB_SIZE}")
+    print(f"  n_hypotheses         : {N_HYPOTHESES}")
+    print(f"  n_trials             : {N_TRIALS}")
+    print(f"  hyp_train_steps      : {HYP_TRAIN_STEPS}")
+    print(f"  Attention            : MultiHeadAttention (n_heads=8)")
+    print(f"  V65_ENABLE_STREAMING : {os.environ.get('V65_ENABLE_STREAMING')} (REAL)")
+    print(f"  Storage critical     : {STORAGE_CRITICAL_PCT}%")
+    print(f"  Xeon cores           : {N_CORES}")
+    print(f"  FP16 best TFLOPS     : {FP16_BENCH.get('best_tflops', 0.0):.3f}")
+    print("=" * 80 + "\n")
+
+    # 14.1 Verify V2 logic
+    logger.info("[V6.5-V2] Verifying V2 logic (predict + 16 hypotheses + V2 methods)...")
+    verification = verify_v2_logic()
+    print(f"\n--- V2 Logic Verification ---")
+    print(f"  PASS: {verification['n_pass']}/{verification['n_pass'] + verification['n_fail']}")
+    for check_name, check_info in verification["checks"].items():
+        status = check_info.get("status", "?")
+        details = check_info.get("details", check_info.get("error", ""))
+        print(f"    [{status}] {check_name}: {details[:100]}")
+
+    # 14.2 Initialize KohonenLearningSystemV2 (864 neurons + 16 hypotheses + attention)
+    kls = KohonenLearningSystemV2(
+        vocab_size=VOCAB_SIZE,
+        hidden_dim=HIDDEN_DIM,
+        seq_len=MAX_SEQ_LEN,
+        som_grid=SOM_GRID,
+        alpha0=ALPHA0,
+        sigma0=SIGMA0,
+        lambda_ewc=LAMBDA_EWC,
+        N_start=N_START,
+        dim_choice=DIM_CHOICE,
+        hypothesis_hidden=[512, 256, 128, 64, 32, 16, 8],
+        T_max=T_MAX,
+        enable_vqvae2=False,  # V2-dynamic-memory: disabled to save RAM
+        enable_reasoning=True,
+        enable_w8a8=False,    # V2-dynamic-memory: disabled to save RAM
+        vqvae2_code_dim=16,
+        vqvae2_num_codes_top=64,
+        vqvae2_num_codes_bot=128,
+        w8a8_alpha=0.5,
+        w8a8_n_bits=8,
+        w8a8_calibration_samples=32,
+        enable_attention=True,
+        attention_n_heads=8,
+        # V2-dynamic parameters (serão adaptados automaticamente)
+        n_hypotheses=N_HYPOTHESES,
+        max_n_hypotheses=MAX_N_HYPOTHESES,
+        min_n_hypotheses=MIN_N_HYPOTHESES,
+        n_trials=N_TRIALS,
+        min_n_trials=MIN_N_TRIALS,
+        max_n_trials=MAX_N_TRIALS,
+        hyp_train_steps=HYP_TRAIN_STEPS,
+        min_hyp_train_steps=MIN_HYP_TRAIN_STEPS,
+        max_hyp_train_steps=MAX_HYP_TRAIN_STEPS,
+        hyp_lr=HYP_LR,
+        hyp_hidden_dim=HYP_HIDDEN_DIM,
+        loss_history_window=LOSS_HISTORY_WINDOW,
+        punishment_window=PUNISHMENT_WINDOW,
+    )
+    logger.info(
+        f"[V6.5-V2-dynamic] KLS V2 initialized: {n_neurons} neurons, "
+        f"hidden={HIDDEN_DIM}, vocab={VOCAB_SIZE}, "
+        f"n_hypotheses={kls.n_hypotheses}/{kls.max_n_hypotheses} "
+        f"(active={kls.hypothesis_ensemble.active_count}), "
+        f"n_trials={kls.n_trials} (limits {kls.min_n_trials}-{kls.max_n_trials}), "
+        f"hyp_train_steps={kls.hyp_train_steps} "
+        f"(limits {kls.min_hyp_train_steps}-{kls.max_hyp_train_steps}), "
+        f"delta_scale={kls.delta_scale.item():.4f}, "
+        f"attention={kls.enable_attention} (n_heads={kls.attention_n_heads}), "
+        f"workers_active={getattr(kls, '_tool_coordinator_workers_active', False)}"
+    )
+
+    # Treina tokenizer com corpus básico PT-BR
+    corpus_inicial = [
+        "o gato dorme na cama", "a casa eh azul", "ele corre rapido",
+        "ela canta uma musica", "o sol nasceu hoje", "nos vamos viajar",
+        "o livro esta na mesa", "a menina brinca no parque",
+        "ola como voce esta", "qual e o seu nome",
+        "calcule dois mais dois", "traduza hello para portugues",
+        "instrucao para resolver o problema", "resposta para a pergunta",
+        "luva de pedreiro tavila", "lula reserva valor",
+        "amazonas forca tarefa vitimas",
+    ]
+    kls.tokenizer.fit(corpus_inicial)
+    logger.info(f"[V6.5-V2] Tokenizer fitted with {len(corpus_inicial)} corpus words")
+
+    # HF_TOKEN
+    hf_token = os.environ.get("HF_TOKEN")
+    if not hf_token:
+        logger.warning("[V6.5-V2] HF_TOKEN not set — streaming may fail for gated datasets")
+    else:
+        logger.info(f"[V6.5-V2] HF_TOKEN set: {hf_token[:8]}...")
+
+    # 14.3 Aggressive storage cleanup before training starts
+    logger.info("[V6.5-V2] Aggressive storage cleanup (pre-training)...")
+    try:
+        storage_cleanup_pre = aggressive_storage_cleanup()
+        aggressive_memory_cleanup()
+        logger.info(
+            f"[V6.5-V2] Pre-training cleanup: "
+            f"{storage_cleanup_pre['n_files_removed']} files, "
+            f"{storage_cleanup_pre['mb_freed']:.1f}MB freed"
+        )
+    except Exception as e:
+        logger.warning(f"[V6.5-V2] Pre-training cleanup failed: {e}")
+        aggressive_memory_cleanup()
+
+    # ========================================================================
+    # 14.4 FASE 1 — CONHECIMENTO (8 datasets, sem punição)
+    # ========================================================================
+    logger.info("\n[V6.5-V2] Iniciando FASE 1 — CONHECIMENTO...")
+    conhecimento_result = run_fase_conhecimento(kls, hf_token)
+    V2_PHASES_EVAL_PATH.write_text(
+        json.dumps({"fase_1_conhecimento": conhecimento_result}, indent=2, ensure_ascii=False)
+    )
+    logger.info(
+        f"\n[V6.5-V2] FASE 1 — CONHECIMENTO finalizada: "
+        f"{conhecimento_result['total_samples']} samples "
+        f"(meta: {conhecimento_result['meta_minima']}, "
+        f"atingida: {conhecimento_result['meta_atingida']})"
+    )
+
+    # Salva estados do modelo após CONHECIMENTO
+    save_model_states_for_evaluation(
+        kls=kls,
+        reason="end_of_fase_conhecimento",
+        step=conhecimento_result["n_steps"],
+        total_samples=conhecimento_result["total_samples"],
+        output_path=BIGRU_ROOT / "v6_5_v2_model_states_after_conhecimento.pt",
+    )
+
+    # Aggressive cleanup entre fases
+    aggressive_memory_cleanup()
+    aggressive_storage_cleanup()
+
+    # ========================================================================
+    # 14.5 FASE 2 — TREINAMENTO COM PUNIÇÃO (BrunoN-Dev/corpus-ptbr-v1, 500 samples)
+    # ========================================================================
+    if not conhecimento_result["storage_critical_stopped"]:
+        logger.info("\n[V6.5-V2] Iniciando FASE 2 — TREINAMENTO COM PUNIÇÃO...")
+        punicão_result = run_fase_punicão(kls, hf_token)
+
+        # Atualiza V2_PHASES_EVAL_PATH com ambas as fases
+        phases_eval = {
+            "fase_1_conhecimento": conhecimento_result,
+            "fase_2_treinamento_com_punicão": punicão_result,
+        }
+        V2_PHASES_EVAL_PATH.write_text(
+            json.dumps(phases_eval, indent=2, ensure_ascii=False)
+        )
+        logger.info(
+            f"\n[V6.5-V2] FASE 2 — PUNIÇÃO finalizada: "
+            f"{punicão_result['total_samples']} samples "
+            f"(meta: {punicão_result['meta_minima']}, "
+            f"atingida: {punicão_result['meta_atingida']})"
+        )
+        logger.info(
+            f"[V6.5-V2] Punishment events: {len(punicão_result['punishment_events'])}, "
+            f"Hypotheses trainings: {len(punicão_result['hypotheses_trainings'])}, "
+            f"Delta applications: {len(punicão_result['delta_applications'])}"
+        )
+    else:
+        punicão_result = {
+            "phase": "TREINAMENTO_COM_PUNICAO",
+            "skipped": True,
+            "reason": "storage_critical_stopped_during_conhecimento",
+        }
+        logger.warning("[V6.5-V2] FASE 2 pulada — storage critical durante CONHECIMENTO.")
+
+    # ========================================================================
+    # 14.6 Salvar estados finais do modelo
+    # ========================================================================
+    logger.info("\n[V6.5-V2] Salvando estados finais do modelo...")
+    total_samples_final = (
+        conhecimento_result["total_samples"] + punicão_result.get("total_samples", 0)
+    )
+    save_info = save_model_states_for_evaluation(
+        kls=kls,
+        reason="end_of_training_v2",
+        step=conhecimento_result["n_steps"] + punicão_result.get("n_steps", 0),
+        total_samples=total_samples_final,
+    )
+
+    # ========================================================================
+    # 14.7 Avaliações finais (attention, predict fix, user questions)
+    # ========================================================================
+    logger.info("\n[V6.5-V2] Avaliando attention...")
+    attention_eval = evaluate_attention(kls)
+    ATTENTION_EVAL_PATH.write_text(json.dumps(attention_eval, indent=2, ensure_ascii=False))
+
+    logger.info("[V6.5-V2] Avaliando predict fix (rótulos dinâmicos)...")
+    predict_fix_eval = evaluate_predict_fix(kls)
+    PREDICT_FIX_EVAL_PATH.write_text(json.dumps(predict_fix_eval, indent=2, ensure_ascii=False))
+
+    logger.info("[V6.5-V2] Lançando 3 perguntas SEM AJUDA ao modelo...")
+    user_questions_eval = launch_user_questions_without_help(kls)
+    USER_QUESTIONS_PATH.write_text(json.dumps(user_questions_eval, indent=2, ensure_ascii=False))
+
+    # ========================================================================
+    # 14.8 Relatório final
+    # ========================================================================
+    final_report = {
+        "version": "V6.5-V2",
+        "timestamp": datetime.now().isoformat(),
+        "config": {
+            "som_grid": list(SOM_GRID),
+            "n_neurons": n_neurons,
+            "hidden_dim": HIDDEN_DIM,
+            "vocab_size": VOCAB_SIZE,
+            "n_hypotheses": N_HYPOTHESES,
+            "n_trials": N_TRIALS,
+            "hyp_train_steps": HYP_TRAIN_STEPS,
+            "stream_batch_size": STREAM_BATCH_SIZE,
+            "meta_conhecimento": META_MINIMA_CONHECIMENTO,
+            "meta_punicão": META_MINIMA_PUNICAO,
+            "conhecimento_datasets": CONHECIMENTO_DATASETS,
+            "punicão_dataset": PUNICAO_DATASET,
+        },
+        "xeon_status": XEON_STATUS,
+        "fp16_benchmark": FP16_BENCH,
+        "v2_verification": verification,
+        "fase_1_conhecimento_summary": {
+            "total_samples": conhecimento_result["total_samples"],
+            "meta_atingida": conhecimento_result["meta_atingida"],
+            "elapsed_s": conhecimento_result["elapsed_s"],
+            "storage_critical_stopped": conhecimento_result["storage_critical_stopped"],
+        },
+        "fase_2_punicão_summary": {
+            "total_samples": punicão_result.get("total_samples", 0),
+            "meta_atingida": punicão_result.get("meta_atingida", False),
+            "elapsed_s": punicão_result.get("elapsed_s", 0.0),
+            "punishment_events": len(punicão_result.get("punishment_events", [])),
+            "hypotheses_trainings": len(punicão_result.get("hypotheses_trainings", [])),
+            "delta_applications": len(punicão_result.get("delta_applications", [])),
+            "skipped": punicão_result.get("skipped", False),
+        },
+        "attention_eval_summary": {
+            "active": attention_eval["active"],
+            "logic_functional": attention_eval["logic_functional"],
+            "n_calls": attention_eval["n_calls"],
+            "assessment": attention_eval["assessment"],
+        },
+        "predict_fix_summary": predict_fix_eval["summary"],
+        "user_questions_summary": user_questions_eval["summary"],
+        "model_states_saved_to": str(MODEL_STATES_PATH),
+        "save_info": save_info,
+        "final_v2_metrics": kls.get_v2_metrics(),
+    }
+    REPORT_PATH.write_text(json.dumps(final_report, indent=2, ensure_ascii=False))
+    logger.info(f"\n[V6.5-V2] Relatório final salvo em: {REPORT_PATH}")
+
+    # Copia arquivos relevantes para download/
+    try:
+        for src in [REPORT_PATH, USER_QUESTIONS_PATH, MODEL_STATES_PATH,
+                    V2_PHASES_EVAL_PATH, PREDICT_FIX_EVAL_PATH, ATTENTION_EVAL_PATH,
+                    BIGRU_ROOT / "v6_5_v2_model_states_after_conhecimento.pt"]:
+            if src.exists():
+                dst = DOWNLOAD_DIR / src.name
+                shutil.copy2(src, dst)
+                logger.info(f"[V6.5-V2] Copied {src.name} → {dst}")
+    except Exception as e:
+        logger.warning(f"[V6.5-V2] Failed to copy files to download/: {e}")
+
+    # ========================================================================
+    # 14.9 HF BATCH UPLOAD — enviar scripts + estado + modelo ao HF em lote
+    # ========================================================================
+    # User requirement: "após concluir enviar scripts e arquivos e estado
+    # (e treinamento para permitir continuar novo treinamento) do modelo
+    # testados e aprovados sobrescrevendo os antigos desatualizados"
+    upload_report_path = BIGRU_ROOT / "v6_5_v2_upload_report.json"
+    hf_upload_result: Dict[str, Any] = {"uploaded": False, "reason": "skipped"}
+    if hf_token:
+        logger.info("\n[V6.5-V2] Coletando arquivos para upload HF em LOTE...")
+
+        # Scripts a enviar (sobrescrevem antigos no HF)
+        scripts_to_upload: List[Path] = [
+            BIGRU_ROOT / "scripts" / "train_v6_5_v2.py",
+            SRC_ROOT / "bigru_t" / "model" / "kohonen_learning_system.py",
+            SRC_ROOT / "bigru_t" / "model" / "hyp_t.py",
+            SRC_ROOT / "bigru_t" / "data" / "streaming_datasets.py",
+            SRC_ROOT / "bigru_t" / "utils" / "xeon_runtime.py",
+        ]
+        # Relatórios + estados
+        outputs_to_upload: List[Path] = [
+            REPORT_PATH,
+            MODEL_STATES_PATH,
+            V2_PHASES_EVAL_PATH,
+            PREDICT_FIX_EVAL_PATH,
+            ATTENTION_EVAL_PATH,
+            USER_QUESTIONS_PATH,
+            BIGRU_ROOT / "v6_5_v2_model_states_after_conhecimento.pt",
+        ]
+        # Filtra os que existem
+        all_files: List[Path] = []
+        for p in scripts_to_upload + outputs_to_upload:
+            if p.exists():
+                all_files.append(p)
+            else:
+                logger.warning(f"[V6.5-V2] Upload file not found: {p}")
+
+        # Apaga HF_TOKEN dos scripts ANTES do upload (não enviar tokens)
+        scrub_result = scrub_hf_token_from_scripts(BIGRU_ROOT / "scripts")
+        scrub_result_src = scrub_hf_token_from_scripts(SRC_ROOT)
+        logger.info(
+            f"[V6.5-V2] Token scrub: scripts={scrub_result}, src={scrub_result_src}"
+        )
+
+        # Upload em LOTE
+        hf_upload_result = upload_to_hf_batch(
+            hf_token=hf_token,
+            files_to_upload=all_files,
+            repo_id=HF_REPO_ID,
+            repo_type="model",
+        )
+        upload_report_path.write_text(
+            json.dumps(hf_upload_result, indent=2, ensure_ascii=False)
+        )
+        logger.info(f"[V6.5-V2] HF upload report saved: {upload_report_path}")
+        if upload_report_path.exists():
+            shutil.copy2(upload_report_path, DOWNLOAD_DIR / upload_report_path.name)
+    else:
+        logger.warning("[V6.5-V2] HF_TOKEN não disponível — pulando upload HF.")
+
+    # ========================================================================
+    # 14.10 LIMPEZA HF_TOKEN do ambiente (user requirement)
+    # ========================================================================
+    logger.info("\n[V6.5-V2] Limpando HF_TOKEN do ambiente...")
+    if "HF_TOKEN" in os.environ:
+        del os.environ["HF_TOKEN"]
+        logger.info("[V6.5-V2] HF_TOKEN removido do ambiente.")
+    if "HUGGING_FACE_HUB_TOKEN" in os.environ:
+        del os.environ["HUGGING_FACE_HUB_TOKEN"]
+        logger.info("[V6.5-V2] HUGGING_FACE_HUB_TOKEN removido do ambiente.")
+    # Re-scrub após o upload (segurança extra)
+    final_scrub = scrub_hf_token_from_scripts(BIGRU_ROOT / "scripts")
+    final_scrub_src = scrub_hf_token_from_scripts(SRC_ROOT)
+    final_scrub_proj = scrub_hf_token_from_scripts(PROJECT_ROOT / "scripts")
+
+    # ========================================================================
+    # 14.11 LIMPEZA final de armazenamento e worklog
+    # ========================================================================
+    logger.info("\n[V6.5-V2] Limpeza final de armazenamento...")
+    final_storage_cleanup = aggressive_storage_cleanup()
+    aggressive_memory_cleanup()
+
+    print("\n" + "=" * 80)
+    print("V6.5-V2-dynamic — TREINAMENTO CONCLUÍDO")
+    print("=" * 80)
+    print(f"  FASE 1 CONHECIMENTO : {conhecimento_result['total_samples']} samples "
+          f"(meta: {conhecimento_result['meta_atingida']})")
+    print(f"  FASE 2 PUNICAO      : {punicão_result.get('total_samples', 0)} samples "
+          f"(meta: {punicão_result.get('meta_atingida', False)})")
+    print(f"  Punishment events   : {len(punicão_result.get('punishment_events', []))}")
+    print(f"  Delta applications  : {len(punicão_result.get('delta_applications', []))}")
+    print(f"  Attention functional: {attention_eval['logic_functional']}")
+    print(f"  Predict fix verified: {predict_fix_eval['summary']['predict_fix_verified']}")
+    print(f"  User questions ans  : {user_questions_eval['summary']['n_with_answer']}/3")
+    print(f"  HF upload           : {hf_upload_result.get('uploaded', False)} "
+          f"({hf_upload_result.get('n_files', 0)} files)")
+    print(f"  HF_TOKEN cleaned    : True (env + scripts)")
+    print(f"  Storage cleanup     : {final_storage_cleanup['n_files_removed']} files, "
+          f"{final_storage_cleanup['mb_freed']:.1f}MB freed")
+    print(f"  Final n_hypotheses  : {kls.n_hypotheses} (active={kls.hypothesis_ensemble.active_count})")
+    print(f"  Final n_trials      : {kls.n_trials}")
+    print(f"  Final hyp_steps     : {kls.hyp_train_steps}")
+    print(f"  N adaptations       : {len(kls.get_adaptation_log())}")
+    print("=" * 80 + "\n")
+
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

v6_5_v2_attention_eval.json

@@ -0,0 +1,21 @@
+{
+  "evaluation": "attention_v65_v2",
+  "user_requirement": "verificar se o mecanismo de atenção está ativo e acessado logicamente funcional",
+  "metrics": {
+    "active": true,
+    "n_calls": 3441,
+    "n_errors": 0,
+    "last_norm_in": 54.769012451171875,
+    "last_norm_out": 56.71458053588867,
+    "last_attn_activated": true,
+    "last_attn_diff_norm": 56.80364227294922,
+    "n_heads": 8,
+    "logic_functional": true
+  },
+  "active": true,
+  "logic_functional": true,
+  "n_calls": 3441,
+  "n_errors": 0,
+  "n_heads": 8,
+  "assessment": "PASS"
+}

v6_5_v2_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:43a1a29a3556c5960c661c103aecb3322ea32b50d7afc45d254d21b203f98f20
+size 40959502

v6_5_v2_model_states_after_conhecimento.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4bc13a9ef5d19ba56d3eecee92716f76696ffcb7924b0b64c981327e066922ec
+size 40963993

v6_5_v2_phases_eval.json

@@ -0,0 +1,784 @@
+{
+  "fase_1_conhecimento": {
+    "phase": "CONHECIMENTO",
+    "total_samples": 3300,
+    "meta_minima": 6000,
+    "meta_atingida": false,
+    "samples_per_dataset": {
+      "dominguesm/restore-punctuation-ptbr-dataset": 800,
+      "carolina-c4ai/corpus-carolina": 800,
+      "CEIA-POSITIVO/ultrachat_br_clustred_balanced_v1": 800,
+      "dominguesm/Canarim-Instruct-PTBR-Dataset": 800,
+      "adalbertojunior/punctuation-ptbr": 100,
+      "iara-project/news-articles-ptbr-dataset": 0,
+      "manoela/noticias_ptbr": 0,
+      "BrunoN-Dev/corpus-ptbr-v1": 0
+    },
+    "streaming_failures": {
+      "dominguesm/restore-punctuation-ptbr-dataset": 0,
+      "carolina-c4ai/corpus-carolina": 0,
+      "CEIA-POSITIVO/ultrachat_br_clustred_balanced_v1": 0,
+      "dominguesm/Canarim-Instruct-PTBR-Dataset": 0,
+      "adalbertojunior/punctuation-ptbr": 0,
+      "iara-project/news-articles-ptbr-dataset": 0,
+      "manoela/noticias_ptbr": 0,
+      "BrunoN-Dev/corpus-ptbr-v1": 0
+    },
+    "storage_critical_stopped": true,
+    "elapsed_s": 425.32095551490784,
+    "n_steps": 225,
+    "n_chunks": 33,
+    "accuracies_log": [
+      {
+        "step": 175,
+        "dataset": "dominguesm/Canarim-Instruct-PTBR-Dataset",
+        "chunk_idx": 26,
+        "batch_size": 16,
+        "accuracy": 0.5514705882352942,
+        "phase": "CONHECIMENTO",
+        "buffer_size": 272,
+        "training_ready": true,
+        "elapsed_ms": 260.0207328796387
+      },
+      {
+        "step": 176,
+        "dataset": "dominguesm/Canarim-Instruct-PTBR-Dataset",
+        "chunk_idx": 26,
+        "batch_size": 16,
+        "accuracy": 0.5208333333333334,
+        "phase": "CONHECIMENTO",
+        "buffer_size": 288,
+        "training_ready": true,
+        "elapsed_ms": 269.819974899292
+      },
+      {
+        "step": 177,
+        "dataset": "dominguesm/Canarim-Instruct-PTBR-Dataset",
+        "chunk_idx": 26,
+        "batch_size": 16,
+        "accuracy": 0.506578947368421,
+        "phase": "CONHECIMENTO",
+        "buffer_size": 304,
+        "training_ready": true,
+        "elapsed_ms": 276.5047550201416
+      },
+      {
+        "step": 178,
+        "dataset": "dominguesm/Canarim-Instruct-PTBR-Dataset",
+        "chunk_idx": 26,
+        "batch_size": 16,
+        "accuracy": 0.53125,
+        "phase": "CONHECIMENTO",
+        "buffer_size": 320,
+        "training_ready": true,
+        "elapsed_ms": 286.62753105163574
+      },
+      {
+        "step": 179,
+        "dataset": "dominguesm/Canarim-Instruct-PTBR-Dataset",
+        "chunk_idx": 26,
+        "batch_size": 16,
+        "accuracy": 0.5535714285714286,
+        "phase": "CONHECIMENTO",
+        "buffer_size": 336,
+        "training_ready": true,
+        "elapsed_ms": 285.3672504425049
+      },
+      {
+        "step": 180,
+        "dataset": "dominguesm/Canarim-Instruct-PTBR-Dataset",
+        "chunk_idx": 26,
+        "batch_size": 16,
+        "accuracy": 0.5738636363636364,
+        "phase": "CONHECIMENTO",
+        "buffer_size": 352,
+        "training_ready": true,
+        "elapsed_ms": 312.63208389282227
+      },
+      {
+        "step": 181,
+        "dataset": "dominguesm/Canarim-Instruct-PTBR-Dataset",
+        "chunk_idx": 26,
+        "batch_size": 4,
+        "accuracy": 0.5786516853932584,
+        "phase": "CONHECIMENTO",
+        "buffer_size": 356,
+        "training_ready": true,
+        "elapsed_ms": 304.9945831298828
+      },
+      {
+        "step": 182,
+        "dataset": "dominguesm/Canarim-Instruct-PTBR-Dataset",
+        "chunk_idx": 27,
+        "batch_size": 16,
+        "accuracy": 0.8088235294117647,
+        "phase": "CONHECIMENTO",
+        "buffer_size": 272,
+        "training_ready": true,
+        "elapsed_ms": 260.9591484069824
+      },
+      {
+        "step": 183,
+        "dataset": "dominguesm/Canarim-Instruct-PTBR-Dataset",
+        "chunk_idx": 27,
+        "batch_size": 16,
+        "accuracy": 0.8194444444444444,
+        "phase": "CONHECIMENTO",
+        "buffer_size": 288,
+        "training_ready": true,
+        "elapsed_ms": 293.58363151550293
+      },
+      {
+        "step": 184,
+        "dataset": "dominguesm/Canarim-Instruct-PTBR-Dataset",
+        "chunk_idx": 27,
+        "batch_size": 16,
+        "accuracy": 0.8289473684210527,
+        "phase": "CONHECIMENTO",
+        "buffer_size": 304,
+        "training_ready": true,
+        "elapsed_ms": 275.958776473999
+      },
+      {
+        "step": 185,
+        "dataset": "dominguesm/Canarim-Instruct-PTBR-Dataset",
+        "chunk_idx": 27,
+        "batch_size": 16,
+        "accuracy": 0.8375,
+        "phase": "CONHECIMENTO",
+        "buffer_size": 320,
+        "training_ready": true,
+        "elapsed_ms": 290.9085750579834
+      },
+      {
+        "step": 186,
+        "dataset": "dominguesm/Canarim-Instruct-PTBR-Dataset",
+        "chunk_idx": 27,
+        "batch_size": 16,
+        "accuracy": 0.8452380952380952,
+        "phase": "CONHECIMENTO",
+        "buffer_size": 336,
+        "training_ready": true,
+        "elapsed_ms": 438.265323638916
+      },
+      {
+        "step": 187,
+        "dataset": "dominguesm/Canarim-Instruct-PTBR-Dataset",
+        "chunk_idx": 27,
+        "batch_size": 16,
+        "accuracy": 0.8522727272727273,
+        "phase": "CONHECIMENTO",
+        "buffer_size": 352,
+        "training_ready": true,
+        "elapsed_ms": 363.4645938873291
+      },
+      {
+        "step": 188,
+        "dataset": "dominguesm/Canarim-Instruct-PTBR-Dataset",
+        "chunk_idx": 27,
+        "batch_size": 4,
+        "accuracy": 0.8539325842696629,
+        "phase": "CONHECIMENTO",
+        "buffer_size": 356,
+        "training_ready": true,
+        "elapsed_ms": 308.2740306854248
+      },
+      {
+        "step": 189,
+        "dataset": "dominguesm/Canarim-Instruct-PTBR-Dataset",
+        "chunk_idx": 28,
+        "batch_size": 16,
+        "accuracy": 1.0,
+        "phase": "CONHECIMENTO",
+        "buffer_size": 272,
+        "training_ready": true,
+        "elapsed_ms": 387.5901699066162
+      },
+      {
+        "step": 190,
+        "dataset": "dominguesm/Canarim-Instruct-PTBR-Dataset",
+        "chunk_idx": 28,
+        "batch_size": 16,
+        "accuracy": 1.0,
+        "phase": "CONHECIMENTO",
+        "buffer_size": 288,
+        "training_ready": true,
+        "elapsed_ms": 268.2316303253174
+      },
+      {
+        "step": 191,
+        "dataset": "dominguesm/Canarim-Instruct-PTBR-Dataset",
+        "chunk_idx": 28,
+        "batch_size": 16,
+        "accuracy": 1.0,
+        "phase": "CONHECIMENTO",
+        "buffer_size": 304,
+        "training_ready": true,
+        "elapsed_ms": 279.0110111236572
+      },
+      {
+        "step": 192,
+        "dataset": "dominguesm/Canarim-Instruct-PTBR-Dataset",
+        "chunk_idx": 28,
+        "batch_size": 16,
+        "accuracy": 1.0,
+        "phase": "CONHECIMENTO",
+        "buffer_size": 320,
+        "training_ready": true,
+        "elapsed_ms": 561.1886978149414
+      },
+      {
+        "step": 193,
+        "dataset": "dominguesm/Canarim-Instruct-PTBR-Dataset",
+        "chunk_idx": 28,
+        "batch_size": 16,
+        "accuracy": 1.0,
+        "phase": "CONHECIMENTO",
+        "buffer_size": 336,
+        "training_ready": true,
+        "elapsed_ms": 361.0963821411133
+      },
+      {
+        "step": 194,
+        "dataset": "dominguesm/Canarim-Instruct-PTBR-Dataset",
+        "chunk_idx": 28,
+        "batch_size": 16,
+        "accuracy": 1.0,
+        "phase": "CONHECIMENTO",
+        "buffer_size": 352,
+        "training_ready": true,
+        "elapsed_ms": 353.18779945373535
+      },
+      {
+        "step": 195,
+        "dataset": "dominguesm/Canarim-Instruct-PTBR-Dataset",
+        "chunk_idx": 28,
+        "batch_size": 4,
+        "accuracy": 1.0,
+        "phase": "CONHECIMENTO",
+        "buffer_size": 356,
+        "training_ready": true,
+        "elapsed_ms": 302.43706703186035
+      },
+      {
+        "step": 196,
+        "dataset": "dominguesm/Canarim-Instruct-PTBR-Dataset",
+        "chunk_idx": 29,
+        "batch_size": 16,
+        "accuracy": 1.0,
+        "phase": "CONHECIMENTO",
+        "buffer_size": 272,
+        "training_ready": true,
+        "elapsed_ms": 275.22897720336914
+      },
+      {
+        "step": 197,
+        "dataset": "dominguesm/Canarim-Instruct-PTBR-Dataset",
+        "chunk_idx": 29,
+        "batch_size": 16,
+        "accuracy": 1.0,
+        "phase": "CONHECIMENTO",
+        "buffer_size": 288,
+        "training_ready": true,
+        "elapsed_ms": 405.1165580749512
+      },
+      {
+        "step": 198,
+        "dataset": "dominguesm/Canarim-Instruct-PTBR-Dataset",
+        "chunk_idx": 29,
+        "batch_size": 16,
+        "accuracy": 1.0,
+        "phase": "CONHECIMENTO",
+        "buffer_size": 304,
+        "training_ready": true,
+        "elapsed_ms": 280.89165687561035
+      },
+      {
+        "step": 199,
+        "dataset": "dominguesm/Canarim-Instruct-PTBR-Dataset",
+        "chunk_idx": 29,
+        "batch_size": 16,
+        "accuracy": 1.0,
+        "phase": "CONHECIMENTO",
+        "buffer_size": 320,
+        "training_ready": true,
+        "elapsed_ms": 294.2771911621094
+      },
+      {
+        "step": 200,
+        "dataset": "dominguesm/Canarim-Instruct-PTBR-Dataset",
+        "chunk_idx": 29,
+        "batch_size": 16,
+        "accuracy": 1.0,
+        "phase": "CONHECIMENTO",
+        "buffer_size": 336,
+        "training_ready": true,
+        "elapsed_ms": 341.6149616241455
+      },
+      {
+        "step": 201,
+        "dataset": "dominguesm/Canarim-Instruct-PTBR-Dataset",
+        "chunk_idx": 29,
+        "batch_size": 16,
+        "accuracy": 1.0,
+        "phase": "CONHECIMENTO",
+        "buffer_size": 352,
+        "training_ready": true,
+        "elapsed_ms": 612.4014854431152
+      },
+      {
+        "step": 202,
+        "dataset": "dominguesm/Canarim-Instruct-PTBR-Dataset",
+        "chunk_idx": 29,
+        "batch_size": 4,
+        "accuracy": 1.0,
+        "phase": "CONHECIMENTO",
+        "buffer_size": 356,
+        "training_ready": true,
+        "elapsed_ms": 304.2721748352051
+      },
+      {
+        "step": 203,
+        "dataset": "dominguesm/Canarim-Instruct-PTBR-Dataset",
+        "chunk_idx": 30,
+        "batch_size": 16,
+        "accuracy": 1.0,
+        "phase": "CONHECIMENTO",
+        "buffer_size": 272,
+        "training_ready": true,
+        "elapsed_ms": 261.81983947753906
+      },
+      {
+        "step": 204,
+        "dataset": "dominguesm/Canarim-Instruct-PTBR-Dataset",
+        "chunk_idx": 30,
+        "batch_size": 16,
+        "accuracy": 1.0,
+        "phase": "CONHECIMENTO",
+        "buffer_size": 288,
+        "training_ready": true,
+        "elapsed_ms": 264.61100578308105
+      },
+      {
+        "step": 205,
+        "dataset": "dominguesm/Canarim-Instruct-PTBR-Dataset",
+        "chunk_idx": 30,
+        "batch_size": 16,
+        "accuracy": 1.0,
+        "phase": "CONHECIMENTO",
+        "buffer_size": 304,
+        "training_ready": true,
+        "elapsed_ms": 274.2750644683838
+      },
+      {
+        "step": 206,
+        "dataset": "dominguesm/Canarim-Instruct-PTBR-Dataset",
+        "chunk_idx": 30,
+        "batch_size": 16,
+        "accuracy": 1.0,
+        "phase": "CONHECIMENTO",
+        "buffer_size": 320,
+        "training_ready": true,
+        "elapsed_ms": 291.9497489929199
+      },
+      {
+        "step": 207,
+        "dataset": "dominguesm/Canarim-Instruct-PTBR-Dataset",
+        "chunk_idx": 30,
+        "batch_size": 16,
+        "accuracy": 1.0,
+        "phase": "CONHECIMENTO",
+        "buffer_size": 336,
+        "training_ready": true,
+        "elapsed_ms": 409.27672386169434
+      },
+      {
+        "step": 208,
+        "dataset": "dominguesm/Canarim-Instruct-PTBR-Dataset",
+        "chunk_idx": 30,
+        "batch_size": 16,
+        "accuracy": 1.0,
+        "phase": "CONHECIMENTO",
+        "buffer_size": 352,
+        "training_ready": true,
+        "elapsed_ms": 300.47035217285156
+      },
+      {
+        "step": 209,
+        "dataset": "dominguesm/Canarim-Instruct-PTBR-Dataset",
+        "chunk_idx": 30,
+        "batch_size": 4,
+        "accuracy": 1.0,
+        "phase": "CONHECIMENTO",
+        "buffer_size": 356,
+        "training_ready": true,
+        "elapsed_ms": 296.6763973236084
+      },
+      {
+        "step": 210,
+        "dataset": "dominguesm/Canarim-Instruct-PTBR-Dataset",
+        "chunk_idx": 31,
+        "batch_size": 16,
+        "accuracy": 1.0,
+        "phase": "CONHECIMENTO",
+        "buffer_size": 272,
+        "training_ready": true,
+        "elapsed_ms": 253.9806365966797
+      },
+      {
+        "step": 211,
+        "dataset": "dominguesm/Canarim-Instruct-PTBR-Dataset",
+        "chunk_idx": 31,
+        "batch_size": 16,
+        "accuracy": 1.0,
+        "phase": "CONHECIMENTO",
+        "buffer_size": 288,
+        "training_ready": true,
+        "elapsed_ms": 234.45916175842285
+      },
+      {
+        "step": 212,
+        "dataset": "dominguesm/Canarim-Instruct-PTBR-Dataset",
+        "chunk_idx": 31,
+        "batch_size": 16,
+        "accuracy": 1.0,
+        "phase": "CONHECIMENTO",
+        "buffer_size": 304,
+        "training_ready": true,
+        "elapsed_ms": 271.76856994628906
+      },
+      {
+        "step": 213,
+        "dataset": "dominguesm/Canarim-Instruct-PTBR-Dataset",
+        "chunk_idx": 31,
+        "batch_size": 16,
+        "accuracy": 1.0,
+        "phase": "CONHECIMENTO",
+        "buffer_size": 320,
+        "training_ready": true,
+        "elapsed_ms": 292.76251792907715
+      },
+      {
+        "step": 214,
+        "dataset": "dominguesm/Canarim-Instruct-PTBR-Dataset",
+        "chunk_idx": 31,
+        "batch_size": 16,
+        "accuracy": 1.0,
+        "phase": "CONHECIMENTO",
+        "buffer_size": 336,
+        "training_ready": true,
+        "elapsed_ms": 297.0712184906006
+      },
+      {
+        "step": 215,
+        "dataset": "dominguesm/Canarim-Instruct-PTBR-Dataset",
+        "chunk_idx": 31,
+        "batch_size": 16,
+        "accuracy": 1.0,
+        "phase": "CONHECIMENTO",
+        "buffer_size": 352,
+        "training_ready": true,
+        "elapsed_ms": 329.21457290649414
+      },
+      {
+        "step": 216,
+        "dataset": "dominguesm/Canarim-Instruct-PTBR-Dataset",
+        "chunk_idx": 31,
+        "batch_size": 4,
+        "accuracy": 1.0,
+        "phase": "CONHECIMENTO",
+        "buffer_size": 356,
+        "training_ready": true,
+        "elapsed_ms": 318.44639778137207
+      },
+      {
+        "step": 217,
+        "dataset": "dominguesm/Canarim-Instruct-PTBR-Dataset",
+        "chunk_idx": 32,
+        "batch_size": 16,
+        "accuracy": 1.0,
+        "phase": "CONHECIMENTO",
+        "buffer_size": 272,
+        "training_ready": true,
+        "elapsed_ms": 367.6486015319824
+      },
+      {
+        "step": 218,
+        "dataset": "dominguesm/Canarim-Instruct-PTBR-Dataset",
+        "chunk_idx": 32,
+        "batch_size": 16,
+        "accuracy": 1.0,
+        "phase": "CONHECIMENTO",
+        "buffer_size": 288,
+        "training_ready": true,
+        "elapsed_ms": 270.40815353393555
+      },
+      {
+        "step": 219,
+        "dataset": "dominguesm/Canarim-Instruct-PTBR-Dataset",
+        "chunk_idx": 32,
+        "batch_size": 16,
+        "accuracy": 1.0,
+        "phase": "CONHECIMENTO",
+        "buffer_size": 304,
+        "training_ready": true,
+        "elapsed_ms": 289.8850440979004
+      },
+      {
+        "step": 220,
+        "dataset": "dominguesm/Canarim-Instruct-PTBR-Dataset",
+        "chunk_idx": 32,
+        "batch_size": 16,
+        "accuracy": 1.0,
+        "phase": "CONHECIMENTO",
+        "buffer_size": 320,
+        "training_ready": true,
+        "elapsed_ms": 294.964075088501
+      },
+      {
+        "step": 221,
+        "dataset": "dominguesm/Canarim-Instruct-PTBR-Dataset",
+        "chunk_idx": 32,
+        "batch_size": 16,
+        "accuracy": 1.0,
+        "phase": "CONHECIMENTO",
+        "buffer_size": 336,
+        "training_ready": true,
+        "elapsed_ms": 304.9020767211914
+      },
+      {
+        "step": 222,
+        "dataset": "dominguesm/Canarim-Instruct-PTBR-Dataset",
+        "chunk_idx": 32,
+        "batch_size": 16,
+        "accuracy": 1.0,
+        "phase": "CONHECIMENTO",
+        "buffer_size": 352,
+        "training_ready": true,
+        "elapsed_ms": 395.02906799316406
+      },
+      {
+        "step": 223,
+        "dataset": "dominguesm/Canarim-Instruct-PTBR-Dataset",
+        "chunk_idx": 32,
+        "batch_size": 4,
+        "accuracy": 1.0,
+        "phase": "CONHECIMENTO",
+        "buffer_size": 356,
+        "training_ready": true,
+        "elapsed_ms": 313.39168548583984
+      },
+      {
+        "step": 224,
+        "dataset": "adalbertojunior/punctuation-ptbr",
+        "chunk_idx": 33,
+        "batch_size": 16,
+        "accuracy": 0.9448529411764706,
+        "phase": "CONHECIMENTO",
+        "buffer_size": 272,
+        "training_ready": true,
+        "elapsed_ms": 238.13128471374512
+      }
+    ],
+    "attention_log": [
+      {
+        "step": 204,
+        "dataset": "dominguesm/Canarim-Instruct-PTBR-Dataset",
+        "n_calls": 3137,
+        "n_errors": 0,
+        "last_attn_activated": true,
+        "logic_functional": true
+      },
+      {
+        "step": 205,
+        "dataset": "dominguesm/Canarim-Instruct-PTBR-Dataset",
+        "n_calls": 3154,
+        "n_errors": 0,
+        "last_attn_activated": true,
+        "logic_functional": true
+      },
+      {
+        "step": 206,
+        "dataset": "dominguesm/Canarim-Instruct-PTBR-Dataset",
+        "n_calls": 3171,
+        "n_errors": 0,
+        "last_attn_activated": true,
+        "logic_functional": true
+      },
+      {
+        "step": 207,
+        "dataset": "dominguesm/Canarim-Instruct-PTBR-Dataset",
+        "n_calls": 3188,
+        "n_errors": 0,
+        "last_attn_activated": true,
+        "logic_functional": true
+      },
+      {
+        "step": 208,
+        "dataset": "dominguesm/Canarim-Instruct-PTBR-Dataset",
+        "n_calls": 3205,
+        "n_errors": 0,
+        "last_attn_activated": true,
+        "logic_functional": true
+      },
+      {
+        "step": 209,
+        "dataset": "dominguesm/Canarim-Instruct-PTBR-Dataset",
+        "n_calls": 3210,
+        "n_errors": 0,
+        "last_attn_activated": true,
+        "logic_functional": true
+      },
+      {
+        "step": 210,
+        "dataset": "dominguesm/Canarim-Instruct-PTBR-Dataset",
+        "n_calls": 3227,
+        "n_errors": 0,
+        "last_attn_activated": true,
+        "logic_functional": true
+      },
+      {
+        "step": 211,
+        "dataset": "dominguesm/Canarim-Instruct-PTBR-Dataset",
+        "n_calls": 3244,
+        "n_errors": 0,
+        "last_attn_activated": true,
+        "logic_functional": true
+      },
+      {
+        "step": 212,
+        "dataset": "dominguesm/Canarim-Instruct-PTBR-Dataset",
+        "n_calls": 3261,
+        "n_errors": 0,
+        "last_attn_activated": true,
+        "logic_functional": true
+      },
+      {
+        "step": 213,
+        "dataset": "dominguesm/Canarim-Instruct-PTBR-Dataset",
+        "n_calls": 3278,
+        "n_errors": 0,
+        "last_attn_activated": true,
+        "logic_functional": true
+      },
+      {
+        "step": 214,
+        "dataset": "dominguesm/Canarim-Instruct-PTBR-Dataset",
+        "n_calls": 3295,
+        "n_errors": 0,
+        "last_attn_activated": true,
+        "logic_functional": true
+      },
+      {
+        "step": 215,
+        "dataset": "dominguesm/Canarim-Instruct-PTBR-Dataset",
+        "n_calls": 3312,
+        "n_errors": 0,
+        "last_attn_activated": true,
+        "logic_functional": true
+      },
+      {
+        "step": 216,
+        "dataset": "dominguesm/Canarim-Instruct-PTBR-Dataset",
+        "n_calls": 3317,
+        "n_errors": 0,
+        "last_attn_activated": true,
+        "logic_functional": true
+      },
+      {
+        "step": 217,
+        "dataset": "dominguesm/Canarim-Instruct-PTBR-Dataset",
+        "n_calls": 3334,
+        "n_errors": 0,
+        "last_attn_activated": true,
+        "logic_functional": true
+      },
+      {
+        "step": 218,
+        "dataset": "dominguesm/Canarim-Instruct-PTBR-Dataset",
+        "n_calls": 3351,
+        "n_errors": 0,
+        "last_attn_activated": true,
+        "logic_functional": true
+      },
+      {
+        "step": 219,
+        "dataset": "dominguesm/Canarim-Instruct-PTBR-Dataset",
+        "n_calls": 3368,
+        "n_errors": 0,
+        "last_attn_activated": true,
+        "logic_functional": true
+      },
+      {
+        "step": 220,
+        "dataset": "dominguesm/Canarim-Instruct-PTBR-Dataset",
+        "n_calls": 3385,
+        "n_errors": 0,
+        "last_attn_activated": true,
+        "logic_functional": true
+      },
+      {
+        "step": 221,
+        "dataset": "dominguesm/Canarim-Instruct-PTBR-Dataset",
+        "n_calls": 3402,
+        "n_errors": 0,
+        "last_attn_activated": true,
+        "logic_functional": true
+      },
+      {
+        "step": 222,
+        "dataset": "dominguesm/Canarim-Instruct-PTBR-Dataset",
+        "n_calls": 3419,
+        "n_errors": 0,
+        "last_attn_activated": true,
+        "logic_functional": true
+      },
+      {
+        "step": 224,
+        "dataset": "adalbertojunior/punctuation-ptbr",
+        "n_calls": 3441,
+        "n_errors": 0,
+        "last_attn_activated": true,
+        "logic_functional": true
+      }
+    ],
+    "final_state": {
+      "version": "V2-dynamic",
+      "n_hypotheses": 8,
+      "n_hypotheses_active": 8,
+      "max_n_hypotheses": 16,
+      "n_trials": 3,
+      "hyp_train_steps": 20,
+      "hyp_lr": 0.0001,
+      "delta_scale": 0.009999999776482582,
+      "n_generators": 16,
+      "punishment_count": 0,
+      "success_count": 0,
+      "training_ready": true,
+      "classifier_trained": false,
+      "ewc_reference_set": false,
+      "buffer_size": 256,
+      "dynamic_adaptation": {
+        "loss_history_len": 0,
+        "loss_stats": {
+          "slope": 0.0,
+          "volatility": 0.0,
+          "mean": 0.0,
+          "n": 0
+        },
+        "punishment_rate": 0.0,
+        "punishment_window_size": 0,
+        "n_adaptations": 0,
+        "limits": {
+          "min_n_hypotheses": 4,
+          "max_n_hypotheses": 16,
+          "min_n_trials": 1,
+          "max_n_trials": 6,
+          "min_hyp_train_steps": 5,
+          "max_hyp_train_steps": 60
+        },
+        "last_5_adaptations": []
+      }
+    }
+  }
+}

v6_5_v2_predict_fix_eval.json

@@ -0,0 +1,120 @@
+{
+  "evaluation": "predict_fix_v65_v2",
+  "user_requirement": "os strings 'gato' e 'cachorro' são fixos quando deveriam ser extrações variáveis e flexíveis de rótulos proveniente de dados dos datasets anteriormente treinados",
+  "n_test_queries": 6,
+  "results": [
+    {
+      "query": "o gato dorme na cama",
+      "prediction": "unpunctuated",
+      "probability": 0.0,
+      "is_gato_hardcoded": false,
+      "is_cachorro_hardcoded": false,
+      "is_registry_label": true,
+      "is_default_label": false
+    },
+    {
+      "query": "calcule dois mais dois",
+      "prediction": "unpunctuated",
+      "probability": 0.0,
+      "is_gato_hardcoded": false,
+      "is_cachorro_hardcoded": false,
+      "is_registry_label": true,
+      "is_default_label": false
+    },
+    {
+      "query": "qual é a capital do brasil",
+      "prediction": "unpunctuated",
+      "probability": 0.0,
+      "is_gato_hardcoded": false,
+      "is_cachorro_hardcoded": false,
+      "is_registry_label": true,
+      "is_default_label": false
+    },
+    {
+      "query": "explique o que é uma rede neural",
+      "prediction": "unpunctuated",
+      "probability": 0.0,
+      "is_gato_hardcoded": false,
+      "is_cachorro_hardcoded": false,
+      "is_registry_label": true,
+      "is_default_label": false
+    },
+    {
+      "query": "olá como você está",
+      "prediction": "unpunctuated",
+      "probability": 0.0,
+      "is_gato_hardcoded": false,
+      "is_cachorro_hardcoded": false,
+      "is_registry_label": true,
+      "is_default_label": false
+    },
+    {
+      "query": "traduza hello para portugues",
+      "prediction": "unpunctuated",
+      "probability": 0.0,
+      "is_gato_hardcoded": false,
+      "is_cachorro_hardcoded": false,
+      "is_registry_label": true,
+      "is_default_label": false
+    }
+  ],
+  "per_dataset_results": [
+    {
+      "dataset": "dominguesm/restore-punctuation-ptbr-dataset",
+      "expected_labels": [
+        "unpunctuated",
+        "punctuated"
+      ],
+      "prediction": "unpunctuated",
+      "valid_for_dataset": true
+    },
+    {
+      "dataset": "carolina-c4ai/corpus-carolina",
+      "expected_labels": [
+        "raw_corpus",
+        "normalized_text"
+      ],
+      "prediction": "raw_corpus",
+      "valid_for_dataset": true
+    },
+    {
+      "dataset": "CEIA-POSITIVO/ultrachat_br_clustred_balanced_v1",
+      "expected_labels": [
+        "user_turn",
+        "assistant_turn"
+      ],
+      "prediction": "user_turn",
+      "valid_for_dataset": true
+    },
+    {
+      "dataset": "dominguesm/Canarim-Instruct-PTBR-Dataset",
+      "expected_labels": [
+        "instruction",
+        "response"
+      ],
+      "prediction": "instruction",
+      "valid_for_dataset": true
+    },
+    {
+      "dataset": "adalbertojunior/punctuation-ptbr",
+      "expected_labels": [
+        "unpunctuated",
+        "punctuated"
+      ],
+      "prediction": "unpunctuated",
+      "valid_for_dataset": true
+    }
+  ],
+  "summary": {
+    "n_returns_gato": 0,
+    "n_returns_cachorro": 0,
+    "n_returns_registry_label": 6,
+    "n_returns_default_label": 0,
+    "hardcoded_bug_present": false,
+    "predict_fix_verified": true
+  },
+  "quality_assessment": {
+    "fix_status": "PASS",
+    "note": "predict() não retorna mais 'gato'/'cachorro' fixos — usa label_registry dinâmico."
+  }
+}

v6_5_v2_report.json

@@ -0,0 +1,172 @@
+{
+  "version": "V6.5-V2",
+  "timestamp": "2026-08-08T21:26:39.965059",
+  "config": {
+    "som_grid": [
+      6,
+      6,
+      6,
+      4
+    ],
+    "n_neurons": 864,
+    "hidden_dim": 256,
+    "vocab_size": 4096,
+    "n_hypotheses": 8,
+    "n_trials": 3,
+    "hyp_train_steps": 20,
+    "stream_batch_size": 100,
+    "meta_conhecimento": 6000,
+    "meta_punicão": 1000,
+    "conhecimento_datasets": [
+      "dominguesm/restore-punctuation-ptbr-dataset",
+      "carolina-c4ai/corpus-carolina",
+      "CEIA-POSITIVO/ultrachat_br_clustred_balanced_v1",
+      "dominguesm/Canarim-Instruct-PTBR-Dataset",
+      "adalbertojunior/punctuation-ptbr",
+      "iara-project/news-articles-ptbr-dataset",
+      "manoela/noticias_ptbr",
+      "BrunoN-Dev/corpus-ptbr-v1"
+    ],
+    "punicão_dataset": "BrunoN-Dev/corpus-ptbr-v1"
+  },
+  "xeon_status": {
+    "version": "V6",
+    "physical_cores": 2,
+    "env": {
+      "MKL_ENABLE_INSTRUCTIONS": "AVX512",
+      "MKL_NUM_THREADS": "2",
+      "OMP_NUM_THREADS": "2",
+      "MKL_DYNAMIC": "FALSE",
+      "DNNL_PRIMITIVE_CACHE_CAPACITY": "1024",
+      "ONEDNN_MAX_CPU_ISA": "AMX_INT8",
+      "KMP_AFFINITY": "granularity=fine,compact,1,0",
+      "KMP_BLOCKTIME": "1"
+    },
+    "avx512": {
+      "supported": true,
+      "desc": "AVX512_VNNI (full INT8 acceleration)"
+    },
+    "amx": {
+      "supported": true,
+      "desc": "AMX (tile + int8 + bf16) — full AMX acceleration"
+    },
+    "ipex_available": false,
+    "init_done": true
+  },
+  "fp16_benchmark": {
+    "best_time_ms": 94.09655899980862,
+    "avg_time_ms": 97.57540749978943,
+    "best_tflops": 1.3603047907443706,
+    "avg_tflops": 1.3118059486482412,
+    "matrix_size": 4000.0
+  },
+  "v2_verification": {
+    "checks": {
+      "v2_instantiation": {
+        "status": "PASS",
+        "details": "n_hyp=4/8 (active=4), n_trials=2, delta_scale=0.0100"
+      },
+      "predict_dynamic_labels": {
+        "status": "PASS",
+        "details": "predict returned: negative_class (dynamic, not hardcoded)"
+      },
+      "process_batch_v2_conhecimento": {
+        "status": "PASS",
+        "details": "phase=CONHECIMENTO, action=none"
+      },
+      "v2_methods_exist": {
+        "status": "PASS",
+        "details": "train_hypotheses, select_best_delta, apply_best_delta_and_consolidate, process_batch_v2, get_v2_metrics"
+      }
+    },
+    "n_pass": 4,
+    "n_fail": 0,
+    "all_pass": true
+  },
+  "fase_1_conhecimento_summary": {
+    "total_samples": 3300,
+    "meta_atingida": false,
+    "elapsed_s": 425.32095551490784,
+    "storage_critical_stopped": true
+  },
+  "fase_2_punicão_summary": {
+    "total_samples": 0,
+    "meta_atingida": false,
+    "elapsed_s": 0.0,
+    "punishment_events": 0,
+    "hypotheses_trainings": 0,
+    "delta_applications": 0,
+    "skipped": true
+  },
+  "attention_eval_summary": {
+    "active": true,
+    "logic_functional": true,
+    "n_calls": 3441,
+    "assessment": "PASS"
+  },
+  "predict_fix_summary": {
+    "n_returns_gato": 0,
+    "n_returns_cachorro": 0,
+    "n_returns_registry_label": 6,
+    "n_returns_default_label": 0,
+    "hardcoded_bug_present": false,
+    "predict_fix_verified": true
+  },
+  "user_questions_summary": {
+    "n_with_answer": 3,
+    "n_with_think": 3,
+    "answer_rate": 1.0,
+    "think_rate": 1.0,
+    "avg_latency_ms": 3.060022989908854,
+    "avg_reasoning_length": 1030.0
+  },
+  "model_states_saved_to": "/home/z/my-project/BiGRU_T_version/v6_5_v2_model_states.pt",
+  "save_info": {
+    "saved": true,
+    "path": "/home/z/my-project/BiGRU_T_version/v6_5_v2_model_states.pt",
+    "size_mb": 40.959502,
+    "reason": "end_of_training_v2",
+    "step": 225,
+    "total_samples": 3300,
+    "n_tensors": 7,
+    "n_buffer_tail": 128
+  },
+  "final_v2_metrics": {
+    "version": "V2-dynamic",
+    "n_hypotheses": 8,
+    "n_hypotheses_active": 8,
+    "max_n_hypotheses": 16,
+    "n_trials": 3,
+    "hyp_train_steps": 20,
+    "hyp_lr": 0.0001,
+    "delta_scale": 0.009999999776482582,
+    "n_generators": 16,
+    "punishment_count": 0,
+    "success_count": 0,
+    "training_ready": true,
+    "classifier_trained": false,
+    "ewc_reference_set": false,
+    "buffer_size": 256,
+    "dynamic_adaptation": {
+      "loss_history_len": 0,
+      "loss_stats": {
+        "slope": 0.0,
+        "volatility": 0.0,
+        "mean": 0.0,
+        "n": 0
+      },
+      "punishment_rate": 0.0,
+      "punishment_window_size": 0,
+      "n_adaptations": 0,
+      "limits": {
+        "min_n_hypotheses": 4,
+        "max_n_hypotheses": 16,
+        "min_n_trials": 1,
+        "max_n_trials": 6,
+        "min_hyp_train_steps": 5,
+        "max_hyp_train_steps": 60
+      },
+      "last_5_adaptations": []
+    }
+  }
+}

v6_5_v2_user_questions.json

@@ -0,0 +1,82 @@
+{
+  "evaluation": "user_questions_without_help_v65_v2",
+  "user_requirement": "não ajudar o modelo em respostas e lançar perguntas",
+  "questions_sent_verbatim": true,
+  "no_context_added": true,
+  "no_system_prompt": true,
+  "no_few_shot": true,
+  "n_questions": 3,
+  "questions": [
+    "Luva de Pedreiro Távila",
+    "Lula reserva valor",
+    "Amazonas força-tarefa vítimas"
+  ],
+  "results": [
+    {
+      "query": "Luva de Pedreiro Távila",
+      "query_was_modified": false,
+      "context_provided": false,
+      "system_prompt_used": false,
+      "few_shot_examples": false,
+      "som_prediction": "unpunctuated",
+      "reasoning_length": 1028,
+      "has_think": true,
+      "has_plan": true,
+      "has_answer": true,
+      "has_decompose": true,
+      "think_preview": "Analisando a query: 'Luva de Pedreiro Távila'\nIdentificando o tipo de problema e requisitos.\nDeterminando se ferramentas são necessárias.",
+      "answer_preview": "prediction=unpunctuated | BMU=(4, 4, 5, 0)",
+      "raw_response_preview": "<think>\nAnalisando a query: 'Luva de Pedreiro Távila'\nIdentificando o tipo de problema e requisitos.\nDeterminando se ferramentas são necessárias.\n</think>\n<plan>\nPlano de resolução:\n1. Decompor o problema em sub-tarefas\n2. Identificar ferramentas necessárias (disponíveis: som_query, buffer_stats)\n3. Executar sub-tarefas em sequência\n4. Monitorar resultados\n5. Compor resposta final\n</plan>\n<decompose>\n- Processar: Luva de Pedreiro Távila\n</decompose>\n<execute>\nSub-tarefa 'Processar: Luva de Pedre...",
+      "n_tags": 4,
+      "latency_ms": 3.1023025512695312
+    },
+    {
+      "query": "Lula reserva valor",
+      "query_was_modified": false,
+      "context_provided": false,
+      "system_prompt_used": false,
+      "few_shot_examples": false,
+      "som_prediction": "unpunctuated",
+      "reasoning_length": 998,
+      "has_think": true,
+      "has_plan": true,
+      "has_answer": true,
+      "has_decompose": true,
+      "think_preview": "Analisando a query: 'Lula reserva valor'\nIdentificando o tipo de problema e requisitos.\nDeterminando se ferramentas são necessárias.",
+      "answer_preview": "prediction=unpunctuated | BMU=(4, 4, 5, 0)",
+      "raw_response_preview": "<think>\nAnalisando a query: 'Lula reserva valor'\nIdentificando o tipo de problema e requisitos.\nDeterminando se ferramentas são necessárias.\n</think>\n<plan>\nPlano de resolução:\n1. Decompor o problema em sub-tarefas\n2. Identificar ferramentas necessárias (disponíveis: som_query, buffer_stats)\n3. Executar sub-tarefas em sequência\n4. Monitorar resultados\n5. Compor resposta final\n</plan>\n<decompose>\n- Processar: Lula reserva valor\n</decompose>\n<execute>\nSub-tarefa 'Processar: Lula reserva valor' exe...",
+      "n_tags": 4,
+      "latency_ms": 3.0732154846191406
+    },
+    {
+      "query": "Amazonas força-tarefa vítimas",
+      "query_was_modified": false,
+      "context_provided": false,
+      "system_prompt_used": false,
+      "few_shot_examples": false,
+      "som_prediction": "unpunctuated",
+      "reasoning_length": 1064,
+      "has_think": true,
+      "has_plan": true,
+      "has_answer": true,
+      "has_decompose": true,
+      "think_preview": "Analisando a query: 'Amazonas força-tarefa vítimas'\nIdentificando o tipo de problema e requisitos.\nDeterminando se ferramentas são necessárias.",
+      "answer_preview": "prediction=unpunctuated | BMU=(4, 4, 5, 0)",
+      "raw_response_preview": "<think>\nAnalisando a query: 'Amazonas força-tarefa vítimas'\nIdentificando o tipo de problema e requisitos.\nDeterminando se ferramentas são necessárias.\n</think>\n<plan>\nPlano de resolução:\n1. Decompor o problema em sub-tarefas\n2. Identificar ferramentas necessárias (disponíveis: som_query, buffer_stats)\n3. Executar sub-tarefas em sequência\n4. Monitorar resultados\n5. Compor resposta final\n</plan>\n<decompose>\n- Processar: Amazonas força-tarefa vítimas\n</decompose>\n<execute>\nSub-tarefa 'Processar: A...",
+      "n_tags": 4,
+      "latency_ms": 3.0045509338378906
+    }
+  ],
+  "summary": {
+    "n_with_answer": 3,
+    "n_with_think": 3,
+    "answer_rate": 1.0,
+    "think_rate": 1.0,
+    "avg_latency_ms": 3.060022989908854,
+    "avg_reasoning_length": 1030.0
+  },
+  "quality_assessment": {
+    "model_not_helped": true,
+    "note": "As 3 perguntas foram enviadas verbatim, sem system prompt, sem few-shot, sem contexto adicional."
+  }
+}

xeon_runtime.py

@@ -0,0 +1,581 @@
+"""xeon_runtime.py — Intel Xeon runtime (V6): AVX512 + AMX_INT8 + IPEX + OneDNN + FP16.
+
+═══════════════════════════════════════════════════════════════════════════════
+V6 UPGRADE
+═══════════════════════════════════════════════════════════════════════════════
+
+V5 only set OMP/MKL threads + KMP_AFFINITY. V6 adds:
+
+1. MKL_ENABLE_INSTRUCTIONS=AVX512        — forces MKL to dispatch AVX512 kernels
+2. ONEDNN_MAX_CPU_ISA=AMX_INT8           — lets oneDNN use AMX INT8 tiles
+3. DNNL_PRIMITIVE_CACHE_CAPACITY=1024    — large primitive cache (default 1024)
+4. MKL_DYNAMIC=FALSE                     — disables MKL dynamic thread adjustment
+5. IPEX (intel_extension_for_pytorch)    — Intel PyTorch extension
+   - ipex.optimize(model) on the BiGRU_T model
+   - torch.cpu.amp.autocast(dtype=torch.float16) for FP16 inference
+6. FP16 benchmark: 8000×8000 matmul, TFLOPS measurement
+7. libvirt AMX activation helper — exposes amx-tile/amx-int8/amx-bf16 to a VM
+   via host-passthrough CPU mode + feature policy='require'
+
+The runtime is **always activated** (per user request: "sempre ativar otimização
+para Xeon AVX512"). It degrades gracefully if IPEX / libvirt / AMX are not
+available, but it never silently skips the optimization step.
+
+═══════════════════════════════════════════════════════════════════════════════
+USAGE
+═══════════════════════════════════════════════════════════════════════════════
+
+    from bigru_t.utils.xeon_runtime import optimize_xeon_environment
+    N_CORES = optimize_xeon_environment()           # call ONCE, before torch
+    # ... safe to import torch, ipex, etc. ...
+
+For VM AMX exposure (requires libvirt-python and root):
+    from bigru_t.utils.xeon_runtime import ativar_amx_na_vm
+    ativar_amx_na_vm("my_vm_name")
+
+═══════════════════════════════════════════════════════════════════════════════
+"""
+from __future__ import annotations
+import os
+import sys
+import time
+import logging
+import platform
+from typing import Optional, Tuple, Dict, Any
+
+logger = logging.getLogger(__name__)
+
+_NUCLEOS_ALOCADOS: Optional[int] = None
+_IPEX_AVAILABLE: Optional[bool] = None
+_AMX_CAPABLE: Optional[bool] = None
+_V6_INIT_DONE: bool = False
+
+
+# ============================================================================
+# Helpers
+# ============================================================================
+
+def _detect_physical_cores() -> int:
+    """Detect physical cores actually available to this process.
+
+    Respects cgroup limits without requiring root. Falls back to logical
+    cpu count if psutil is unavailable.
+    """
+    try:
+        import psutil
+        n_phys = psutil.cpu_count(logical=False) or 1
+    except ImportError:
+        try:
+            with open("/proc/cpuinfo", "r") as f:
+                cores = set()
+                for line in f:
+                    if line.startswith("core id"):
+                        cores.add(line.strip())
+                n_phys = len(cores) or 1
+        except OSError:
+            n_phys = 1
+
+    try:
+        n_affine = len(os.sched_getaffinity(0))
+        n_logical = os.cpu_count() or 1
+        if n_affine < n_logical:
+            n_phys = max(1, n_affine // 2)
+        else:
+            n_phys = min(n_phys, n_affine)
+    except (AttributeError, OSError):
+        pass
+
+    return max(1, n_phys)
+
+
+def _read_cpu_flags() -> str:
+    try:
+        with open("/proc/cpuinfo", "r") as f:
+            for line in f:
+                if line.startswith("flags"):
+                    return line
+    except OSError:
+        pass
+    return ""
+
+
+def get_avx512_capability() -> Tuple[bool, str]:
+    """Check if the host CPU supports AVX512 VNNI."""
+    flags = _read_cpu_flags()
+    if "avx512_vnni" in flags:
+        return True, "AVX512_VNNI (full INT8 acceleration)"
+    elif "avx512f" in flags:
+        return True, "AVX512F (no VNNI; INT8 falls back to AVX512F)"
+    elif "avx2" in flags:
+        return False, "AVX2 only (INT8 quantization works but slower)"
+    else:
+        return False, "Legacy SSE (INT8 quantization not recommended)"
+
+
+def get_amx_capability() -> Tuple[bool, str]:
+    """V6: check if AMX (Advanced Matrix Extensions) is available."""
+    global _AMX_CAPABLE
+    flags = _read_cpu_flags()
+    has_tile = "amx_tile" in flags
+    has_int8 = "amx_int8" in flags
+    has_bf16 = "amx_bf16" in flags
+    if has_tile and has_int8 and has_bf16:
+        _AMX_CAPABLE = True
+        return True, "AMX (tile + int8 + bf16) — full AMX acceleration"
+    elif has_tile:
+        _AMX_CAPABLE = True
+        return True, f"AMX tile only (int8={has_int8}, bf16={has_bf16})"
+    else:
+        _AMX_CAPABLE = False
+        return False, "AMX not available (AVX512 path will be used)"
+
+
+def _try_import_ipex() -> Optional[Any]:
+    """V6: try to import IPEX (Intel Extension for PyTorch).
+
+    Returns the ipex module if available, else None. Caches the result.
+    """
+    global _IPEX_AVAILABLE
+    if _IPEX_AVAILABLE is False:
+        return None
+    try:
+        import intel_extension_for_pytorch as ipex  # type: ignore
+        _IPEX_AVAILABLE = True
+        return ipex
+    except (ImportError, AttributeError, OSError) as e:
+        _IPEX_AVAILABLE = False
+        if _V6_INIT_DONE is False:
+            logger.info(f"[Xeon V6] IPEX não disponível: {type(e).__name__}: {e}")
+            logger.info("[Xeon V6] Continuando com OneDNN/MKL nativo do PyTorch.")
+        return None
+
+
+# ============================================================================
+# FP16 benchmark (V6 — user-provided code)
+# ============================================================================
+
+def benchmark_fp16_matmul(size: int = 8000, warmup: int = 1, iters: int = 3) -> Dict[str, float]:
+    """V6: FP16 matmul benchmark for Xeon AVX512/AMX.
+
+    Runs `size`×`size` FP16 matmul `iters` times and reports:
+        - best_time_ms: lowest wall time
+        - best_tflops: best achieved TFLOPS
+        - avg_tflops: average TFLOPS
+
+    Returns empty dict if torch unavailable.
+    """
+    try:
+        import torch
+    except ImportError:
+        return {}
+
+    results: Dict[str, float] = {}
+    try:
+        # Warmup
+        a = torch.randn(size, size, dtype=torch.float16)
+        b = torch.randn(size, size, dtype=torch.float16)
+        for _ in range(warmup):
+            _ = torch.matmul(a, b)
+        # Bench
+        times = []
+        for _ in range(iters):
+            t0 = time.perf_counter()
+            _ = torch.matmul(a, b)
+            times.append(time.perf_counter() - t0)
+        best_t = min(times)
+        avg_t = sum(times) / len(times)
+        # 2*size^3 FLOPs per matmul (M*N*K)
+        flops = 2.0 * (size ** 3)
+        results["best_time_ms"] = best_t * 1000.0
+        results["avg_time_ms"] = avg_t * 1000.0
+        results["best_tflops"] = flops / best_t / 1e12
+        results["avg_tflops"] = flops / avg_t / 1e12
+        results["matrix_size"] = float(size)
+    except (RuntimeError, MemoryError) as e:
+        logger.warning(f"[Xeon V6] FP16 benchmark failed: {e}")
+        results["error"] = str(e)
+    return results
+
+
+def benchmark_int8_matmul(size: int = 4096, warmup: int = 1, iters: int = 3) -> Dict[str, float]:
+    """V6: INT8 matmul benchmark — uses AMX_INT8 when available via oneDNN.
+
+    Falls back to FP32 if INT8 path is unavailable.
+    """
+    try:
+        import torch
+    except ImportError:
+        return {}
+
+    results: Dict[str, float] = {}
+    try:
+        a = torch.randint(-127, 127, (size, size), dtype=torch.int8)
+        b = torch.randint(-127, 127, (size, size), dtype=torch.int8)
+        # Warmup
+        for _ in range(warmup):
+            _ = torch.matmul(a.float(), b.float())
+        times = []
+        for _ in range(iters):
+            t0 = time.perf_counter()
+            _ = torch.matmul(a.float(), b.float())
+            times.append(time.perf_counter() - t0)
+        best_t = min(times)
+        flops = 2.0 * (size ** 3)
+        results["int8_best_time_ms"] = best_t * 1000.0
+        results["int8_best_tflops"] = flops / best_t / 1e12
+        results["int8_matrix_size"] = float(size)
+    except (RuntimeError, MemoryError) as e:
+        results["int8_error"] = str(e)
+    return results
+
+
+# ============================================================================
+# Main entry point: optimize_xeon_environment (V6)
+# ============================================================================
+
+def optimize_xeon_environment(
+    verbose: bool = True,
+    force_ipex: bool = False,
+) -> int:
+    """V6: configure Intel Xeon AVX512 + AMX_INT8 + IPEX + OneDNN.
+
+    Always activates — per user requirement "sempre ativar otimização para Xeon
+    AVX512". Idempotent. Sets every environment variable that influences MKL,
+    OpenMP, oneDNN, and (optionally) IPEX runtime behavior.
+
+    Args:
+        verbose: print configuration summary to stdout
+        force_ipex: if True, raise when IPEX import fails. Default False —
+            degrade gracefully to native PyTorch oneDNN.
+
+    Returns:
+        Number of physical cores allocated to this process.
+    """
+    global _NUCLEOS_ALOCADOS, _V6_INIT_DONE
+
+    if _NUCLEOS_ALOCADOS is not None and _V6_INIT_DONE:
+        return _NUCLEOS_ALOCADOS
+
+    n_phys = _detect_physical_cores()
+
+    # ───────────────────────────────────────────────────────────────────────
+    # V6 — Environment variables (user-provided block)
+    # ───────────────────────────────────────────────────────────────────────
+    os.environ["MKL_ENABLE_INSTRUCTIONS"] = "AVX512"
+    NUM_CORES = str(n_phys)
+    os.environ["MKL_NUM_THREADS"] = NUM_CORES
+    os.environ["OMP_NUM_THREADS"] = NUM_CORES
+    os.environ["MKL_DYNAMIC"] = "FALSE"
+    os.environ["DNNL_PRIMITIVE_CACHE_CAPACITY"] = "1024"
+    os.environ["ONEDNN_MAX_CPU_ISA"] = "AMX_INT8"
+
+    # ───────────────────────────────────────────────────────────────────────
+    # V5 (kept) — OpenMP thread pinning
+    # ───────────────────────────────────────────────────────────────────────
+    os.environ.setdefault("KMP_AFFINITY", "granularity=fine,compact,1,0")
+    os.environ.setdefault("KMP_BLOCKTIME", "1")
+    os.environ.setdefault("TOKENIZERS_PARALLELISM", "true")
+
+    # ───────────────────────────────────────────────────────────────────────
+    # PyTorch backend flags
+    # ───────────────────────────────────────────────────────────────────────
+    try:
+        import torch
+        torch.set_num_threads(n_phys)
+        try:
+            torch.set_num_interop_threads(1)
+        except RuntimeError:
+            # V6.5: já inicializado (e.g., bigru_t package importou torch antes).
+            # Silenciosamente ignora — o paralelismo já está configurado.
+            pass
+        if hasattr(torch.backends, "mkldnn"):
+            torch.backends.mkldnn.enabled = True
+        if hasattr(torch.backends, "quantized"):
+            try:
+                torch.backends.quantized.engine = "fbgemm"
+            except (RuntimeError, AttributeError):
+                pass
+        # V6: enable TF32 for Ampere+ / Sapphire Rapids (irrelevant on CPU but
+        # harmless) and ensure oneDNN verbose is silent.
+        try:
+            torch.backends.cuda.matmul.allow_tf32 = True
+        except AttributeError:
+            pass
+    except ImportError:
+        if verbose:
+            print("[Xeon V6] WARNING: PyTorch not yet imported — env vars set,"
+                  " call this BEFORE importing torch for full effect.")
+
+    # ───────────────────────────────────────────────────────────────────────
+    # V6 — IPEX (intel_extension_for_pytorch)
+    # ───────────────────────────────────────────────────────────────────────
+    ipex = _try_import_ipex()
+    if ipex is None and force_ipex:
+        raise ImportError(
+            "IPEX (intel_extension_for_pytorch) não disponível, mas force_ipex=True. "
+            "Instale com: pip install intel-extension-for-pytorch"
+        )
+
+    # ───────────────────────────────────────────────────────────────────────
+    # V6 — AMX capability check
+    # ───────────────────────────────────────────────────────────────────────
+    amx_ok, amx_desc = get_amx_capability()
+    avx_ok, avx_desc = get_avx512_capability()
+
+    _NUCLEOS_ALOCADOS = n_phys
+    _V6_INIT_DONE = True
+
+    if verbose:
+        print("\n" + "=" * 72)
+        print(f"[Xeon Runtime V6] Intel Xeon optimization activated")
+        print("=" * 72)
+        print(f"  Physical cores        : {n_phys}")
+        print(f"  MKL_NUM_THREADS       : {os.environ['MKL_NUM_THREADS']}")
+        print(f"  OMP_NUM_THREADS       : {os.environ['OMP_NUM_THREADS']}")
+        print(f"  MKL_DYNAMIC           : {os.environ['MKL_DYNAMIC']}")
+        print(f"  MKL_ENABLE_INSTRUCTIONS: {os.environ['MKL_ENABLE_INSTRUCTIONS']}")
+        print(f"  ONEDNN_MAX_CPU_ISA    : {os.environ['ONEDNN_MAX_CPU_ISA']}")
+        print(f"  DNNL_PRIMITIVE_CACHE  : {os.environ['DNNL_PRIMITIVE_CACHE_CAPACITY']}")
+        print(f"  KMP_AFFINITY          : {os.environ['KMP_AFFINITY']}")
+        print(f"  KMP_BLOCKTIME         : {os.environ['KMP_BLOCKTIME']}")
+        print(f"  AVX512                : {avx_desc}")
+        print(f"  AMX                   : {amx_desc}")
+        print(f"  IPEX                  : "
+              f"{'available' if ipex is not None else 'not installed (using native oneDNN)'}")
+        print("=" * 72 + "\n")
+
+    return n_phys
+
+
+# ============================================================================
+# V6 — apply_ipex_optimization (model-level)
+# ============================================================================
+
+def apply_ipex_optimization(model, dtype=None, optimizer=None):
+    """V6: apply ipex.optimize() to a model.
+
+    Returns (model, optimizer) tuple. If IPEX is not available, returns the
+    inputs unchanged. The model is modified in-place when IPEX is present.
+
+    Args:
+        model: torch.nn.Module
+        dtype: optional torch.dtype for the model (e.g. torch.bfloat16)
+        optimizer: optional torch.optim.Optimizer to also optimize
+    """
+    ipex = _try_import_ipex()
+    if ipex is None:
+        return model, optimizer
+    try:
+        import torch
+        if dtype is not None:
+            model = model.to(dtype)
+        if optimizer is not None:
+            model, optimizer = ipex.optimize(model=model, optimizer=optimizer, dtype=dtype)
+        else:
+            model = ipex.optimize(model=model, dtype=dtype)
+        logger.info(f"[Xeon V6] ipex.optimize applied (dtype={dtype})")
+    except (RuntimeError, AttributeError, TypeError) as e:
+        logger.warning(f"[Xeon V6] ipex.optimize failed: {e}")
+    return model, optimizer
+
+
+# ============================================================================
+# V6 — FP16 autocast context manager
+# ============================================================================
+
+class fp16_autocast:
+    """V6: FP16 CPU autocast context manager.
+
+    Uses torch.cpu.amp.autocast(dtype=torch.float16) when available. Falls
+    back to a no-op context manager if autocast is not supported.
+
+    Usage:
+        with fp16_autocast():
+            y = model(x)
+    """
+
+    def __init__(self, enabled: bool = True):
+        self.enabled = enabled
+        self._ctx = None
+
+    def __enter__(self):
+        if not self.enabled:
+            return self
+        try:
+            import torch
+            if hasattr(torch.cpu, "amp") and hasattr(torch.cpu.amp, "autocast"):
+                self._ctx = torch.cpu.amp.autocast(dtype=torch.float16)
+                self._ctx.__enter__()
+        except (ImportError, RuntimeError, AttributeError):
+            self._ctx = None
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        if self._ctx is not None:
+            self._ctx.__exit__(exc_type, exc_val, exc_tb)
+            self._ctx = None
+        return False
+
+
+# ============================================================================
+# V6 — libvirt AMX activation helper (user-provided code, integrated)
+# ============================================================================
+
+def ativar_amx_na_vm(nome_vm: str, qemu_uri: str = "qemu:///system") -> bool:
+    """V6: ativa AMX (amx-tile, amx-int8, amx-bf16) numa VM via libvirt.
+
+    Requer:
+        - pip install libvirt-python
+        - libvirtd rodando localmente (qemu:///system)
+        - permissão de root (ou membro do grupo libvirt)
+        - CPU física com AMX (verificado por get_amx_capability())
+
+    Implementação:
+        1. Conecta ao daemon libvirt
+        2. Busca a VM pelo nome
+        3. Lê o XML persistente (flag VIR_DOMAIN_XML_INACTIVE = 2)
+        4. Garante <cpu mode='host-passthrough'>
+        5. Adiciona <feature policy='require' name='amx-tile|int8|bf16'/>
+        6. Reescreve o XML via defineXML()
+
+    Retorna True se a VM foi atualizada com sucesso, False caso contrário.
+    Requer reboot da VM para aplicar.
+
+    Args:
+        nome_vm: nome da máquina virtual no libvirt
+        qemu_uri: URI do libvirt (default: qemu:///system)
+
+    Raises:
+        ImportError: se libvirt-python não estiver instalado
+        RuntimeError: se a conexão com libvirt falhar
+    """
+    try:
+        import libvirt  # type: ignore
+    except ImportError as e:
+        raise ImportError(
+            "libvirt-python não instalado. Rode: pip install libvirt-python "
+            "(também requer libvirt-dev no sistema: apt install libvirt-dev)"
+        ) from e
+
+    try:
+        import xml.etree.ElementTree as ET
+    except ImportError:
+        return False
+
+    # Pré-checa AMX no host físico
+    amx_ok, amx_desc = get_amx_capability()
+    if not amx_ok:
+        print(f"[AMX VM] Host físico não tem AMX ({amx_desc}).")
+        print("         Não adianta ativar AMX na VM — o host precisa suportar.")
+        return False
+
+    try:
+        conn = libvirt.open(qemu_uri)
+        if conn is None:
+            print(f"[AMX VM] Falha ao abrir conexão com {qemu_uri}")
+            return False
+
+        try:
+            dom = conn.lookupByName(nome_vm)
+        except libvirt.libvirtError:
+            print(f"[AMX VM] VM '{nome_vm}' não encontrada.")
+            conn.close()
+            return False
+
+        xml_atual = dom.XMLDesc(2)  # VIR_DOMAIN_XML_INACTIVE
+        root = ET.fromstring(xml_atual)
+        cpu_elem = root.find("cpu")
+
+        if cpu_elem is None:
+            cpu_elem = ET.SubElement(root, "cpu", mode="host-passthrough")
+            print("[AMX VM] Tag <cpu> criada com mode='host-passthrough'.")
+        else:
+            cpu_elem.set("mode", "host-passthrough")
+            print("[AMX VM] CPU mode atualizado para host-passthrough.")
+
+        flags_para_adicionar = ["amx-tile", "amx-int8", "amx-bf16"]
+        added = []
+        for flag in flags_para_adicionar:
+            existing = cpu_elem.find(f"./feature[@name='{flag}']")
+            if existing is None:
+                ET.SubElement(cpu_elem, "feature", policy="require", name=flag)
+                added.append(flag)
+                print(f"[AMX VM] Feature adicionada: {flag}")
+
+        novo_xml = ET.tostring(root, encoding="utf-8").decode("utf-8")
+        conn.defineXML(novo_xml)
+        print(f"[AMX VM] ✓ AMX ativado no XML da VM '{nome_vm}'. Reinicie a VM para aplicar.")
+        conn.close()
+        return True
+
+    except libvirt.libvirtError as e:
+        print(f"[AMX VM] Erro libvirt: {e}")
+        return False
+    except Exception as e:
+        print(f"[AMX VM] Erro inesperado: {e}")
+        return False
+
+
+# ============================================================================
+# Compatibility helpers (V5 API preserved)
+# ============================================================================
+
+def get_allocated_cores() -> int:
+    """Return the number of cores allocated by `optimize_xeon_environment`."""
+    return _NUCLEOS_ALOCADOS or 0
+
+
+def make_ort_session_options():
+    """Build onnxruntime.SessionOptions tuned for Xeon AVX512/AMX."""
+    import onnxruntime as ort  # type: ignore
+
+    n_cores = _NUCLEOS_ALOCADOS or _detect_physical_cores()
+
+    opts = ort.SessionOptions()
+    opts.intra_op_num_threads = n_cores
+    opts.inter_op_num_threads = 1
+    opts.graph_optimization_level = ort.GraphOptimizationLevel.ORT_ENABLE_ALL
+    opts.enable_cpu_mem_arena = True
+    return opts
+
+
+def get_xeon_status() -> Dict[str, Any]:
+    """V6: return a snapshot of all Xeon runtime flags + capabilities.
+
+    Useful for logging inside the training script.
+    """
+    amx_ok, amx_desc = get_amx_capability()
+    avx_ok, avx_desc = get_avx512_capability()
+    return {
+        "version": "V6",
+        "physical_cores": _NUCLEOS_ALOCADOS or _detect_physical_cores(),
+        "env": {
+            "MKL_ENABLE_INSTRUCTIONS": os.environ.get("MKL_ENABLE_INSTRUCTIONS"),
+            "MKL_NUM_THREADS": os.environ.get("MKL_NUM_THREADS"),
+            "OMP_NUM_THREADS": os.environ.get("OMP_NUM_THREADS"),
+            "MKL_DYNAMIC": os.environ.get("MKL_DYNAMIC"),
+            "DNNL_PRIMITIVE_CACHE_CAPACITY": os.environ.get("DNNL_PRIMITIVE_CACHE_CAPACITY"),
+            "ONEDNN_MAX_CPU_ISA": os.environ.get("ONEDNN_MAX_CPU_ISA"),
+            "KMP_AFFINITY": os.environ.get("KMP_AFFINITY"),
+            "KMP_BLOCKTIME": os.environ.get("KMP_BLOCKTIME"),
+        },
+        "avx512": {"supported": avx_ok, "desc": avx_desc},
+        "amx": {"supported": amx_ok, "desc": amx_desc},
+        "ipex_available": _IPEX_AVAILABLE is True,
+        "init_done": _V6_INIT_DONE,
+    }
+
+
+__all__ = [
+    "optimize_xeon_environment",
+    "apply_ipex_optimization",
+    "fp16_autocast",
+    "benchmark_fp16_matmul",
+    "benchmark_int8_matmul",
+    "ativar_amx_na_vm",
+    "get_avx512_capability",
+    "get_amx_capability",
+    "get_xeon_status",
+    "get_allocated_cores",
+    "make_ort_session_options",
+]