V6.7: upload scripts/train_v6_5_v2.py (som_auto_adjust_runner + train script V6.7)

scripts/train_v6_5_v2.py

@@ -230,6 +230,17 @@ ALPHA0 = 0.5
 SIGMA0 = 2.0
 DIM_CHOICE = "y"
 
+# V6.6 — User requirement: "aumentar o truncamento de textos (line 993 do
+# train script) de 200 para 1000+ chars, permitindo mais diversidade de
+# pares byte-level no BBPE".
+# Justificativa matemática: com 200 chars, o BBPE byte-level produz ~200
+# tokens (1 token/byte), limitando a diversidade de pares observados pelo
+# algoritmo de merges. Com 1000+ chars, a diversidade de pares byte-level
+# aumenta ~5×, permitindo que o BBPE aprenda merges mais representativos
+# e efetivamente utilize o vocabulário alvo de 16384 tokens (em vez de
+# ficar limitado a ~293 palavras únicas como na versão word-level).
+TEXT_TRUNCATION_CHARS = 1000
+
 # V2-dynamic — HypothesisEnsemble parameters (valores canônicos restaurados)
 N_HYPOTHESES = 16                  # V6.5-V2-metrics-FIX-3: restored from 8
 MAX_N_HYPOTHESES = 32              # V6.5-V2-metrics-FIX-3: restored from 16
@@ -990,7 +1001,10 @@ def stream_dataset_in_chunks(
                 continue
 
             if sample.raw_text and len(sample.raw_text.strip()) > 0:
-                chunk.append(sample.raw_text.strip()[:200])
+                # V6.6 — User requirement: "aumentar o truncamento de textos
+                # (line 993 do train script) de 200 para 1000+ chars, permitindo
+                # mais diversidade de pares byte-level no BBPE".
+                chunk.append(sample.raw_text.strip()[:TEXT_TRUNCATION_CHARS])
             if len(chunk) >= chunk_size:
                 yield chunk
                 yielded_total += len(chunk)
@@ -1217,7 +1231,13 @@ def run_fase_conhecimento(
     # fixo (16384 rows), então qualquer crescimento de vocab é acomodado.
     # O SOM adapta-se automaticamente às novas projeções 4D do embedding.
     tokenizer_corpus_buffer: List[str] = []
-    TOKENIZER_REFIT_INTERVAL_SAMPLES = 1000  # refit a cada 1000 amostras
+    # V6.7: REATIVADO refit com serial mode + memory guard (fix BBPE OOM)
+    # User requirement: "processar aprimoramento (matemático e lógico) para
+    # resolver: tokenizer-growth refit (was causing crashes during BBPE
+    # parallel training at 1000-sample mark)".
+    # Prova 14: agora serial mode (no fork) + memory guard (skip se RSS>85%)
+    # tornam o refit seguro. Era 10**9 (desativado); agora 1000 (reativado).
+    TOKENIZER_REFIT_INTERVAL_SAMPLES = 1000
     TOKENIZER_CORPUS_CAP = 2000              # cap para evitar OOM (memória: ~2MB)
     last_tokenizer_refit_sample_count = 0
     tokenizer_growth_log: List[Dict[str, Any]] = []
@@ -1511,10 +1531,66 @@ def run_fase_conhecimento(
                     samples_since_refit = total_so_far_now - last_tokenizer_refit_sample_count
                     if (samples_since_refit >= TOKENIZER_REFIT_INTERVAL_SAMPLES
                             and len(tokenizer_corpus_buffer) >= 200):
+                        # V6.7 — MEMORY GUARD before tokenizer refit
+                        # User requirement: "tokenizer-growth refit (was causing
+                        # crashes during BBPE parallel training at 1000-sample mark)".
+                        # Prova 14: ProcessPoolExecutor fork duplica RSS do processo
+                        # pai (modelo + tensores + buffers). Se RSS > 85% do cgroup
+                        # limit, SKIP refit (non-fatal) para evitar OOM-killer.
+                        import os as _os_mod_v67
+                        try:
+                            with open("/proc/self/status") as _f_v67:
+                                _rss_line_v67 = [l for l in _f_v67 if l.startswith("VmRSS:")]
+                                _rss_kb_v67 = int(_rss_line_v67[0].split()[1]) if _rss_line_v67 else 0
+                            _rss_mb_v67 = _rss_kb_v67 / 1024.0
+                            # cgroup limit
+                            _cg_limit_mb_v67 = 4096.0  # default fallback
+                            try:
+                                with open("/sys/fs/cgroup/memory.max") as _f_cg_v67:
+                                    _cg_val_v67 = _f_cg_v67.read().strip()
+                                    if _cg_val_v67 and _cg_val_v67 != "max":
+                                        _cg_limit_mb_v67 = int(_cg_val_v67) / 1024 / 1024
+                            except Exception:
+                                pass  # fallback default
+                            _rss_pct_v67 = _rss_mb_v67 / _cg_limit_mb_v67 if _cg_limit_mb_v67 > 0 else 0
+                            logger.info(
+                                f"[V6.7-memory-guard] RSS={_rss_mb_v67:.0f}MB "
+                                f"({100*_rss_pct_v67:.1f}% of cgroup "
+                                f"{_cg_limit_mb_v67:.0f}MB)"
+                            )
+                            if _rss_pct_v67 > 0.85:
+                                logger.warning(
+                                    f"[V6.7-memory-guard] SKIP refit: RSS "
+                                    f"{100*_rss_pct_v67:.1f}% > 85% of cgroup "
+                                    f"(would trigger OOM via fork). "
+                                    f"gc.collect() e prosseguir sem refit."
+                                )
+                                gc.collect()
+                                if hasattr(torch, "cpu") and hasattr(torch.cpu, "empty_cache"):
+                                    try: torch.cpu.empty_cache()
+                                    except Exception: pass
+                                last_tokenizer_refit_sample_count = total_so_far_now
+                                tokenizer_growth_log.append({
+                                    "step": step,
+                                    "chunk_idx": chunk_idx_global,
+                                    "total_samples": total_so_far_now,
+                                    "skipped_reason": "rss_exceeded_85pct",
+                                    "rss_mb": _rss_mb_v67,
+                                    "cgroup_limit_mb": _cg_limit_mb_v67,
+                                })
+                                # SKIP refit — continue para próximo chunk
+                                raise _SkipRefitV67()
+                        except _SkipRefitV67:
+                            pass  # já tratado acima
+                        except Exception as _guard_err_v67:
+                            logger.warning(
+                                f"[V6.7-memory-guard] guard failed (non-fatal): {_guard_err_v67}"
+                            )
+
                         # Refit tokenizer com corpus acumulado
                         vocab_before = int(getattr(kls.tokenizer, "vocab_size", 0))
                         try:
-                            kls.tokenizer.fit(list(tokenizer_corpus_buffer))
+                            kls.tokenizer.fit(list(tokenizer_corpus_buffer), min_frequency=2)
                             vocab_after = int(getattr(kls.tokenizer, "vocab_size", 0))
                             growth = vocab_after - vocab_before
                             tokenizer_growth_log.append({