V6.5-7ds cleanup: remove legacy v6_5_final_report.json

v6_5_final_report.json

@@ -1,663 +0,0 @@
-{
-  "version": "V6.5-final-real-streaming",
-  "timestamp": "2026-08-08T00:27:22.098795",
-  "user_requirements_checklist": {
-    "HF_TOKEN_deleted_after_use": "PENDING (will delete after upload)",
-    "streaming_datasets_active_REAL": true,
-    "xeon_runtime_active": true,
-    "V65_ENABLE_STREAMING_forced": true,
-    "removed_pre_v64_modules": true,
-    "removed_pre_v64_scripts": true,
-    "kohonen_refactored_removed": true,
-    "kohonen_learning_system_moved_up": true,
-    "vqvae2_active_in_compression_pipeline": true,
-    "reasoning_engine_integrated_to_kls": true,
-    "smoothquant_w8a8_integrated_to_kls": true,
-    "tool_coordinator_workers_reactivated": true,
-    "ewc_w8a8_dequant_benchmark_active": true,
-    "logic_and_bugfixes_verified": true,
-    "exhausted_4_datasets": true,
-    "metrics_reasoning_response_verified": true,
-    "som_grid_864_neurons": true,
-    "mtp_head_size_increased_K6": true,
-    "no_synthetic_data": true,
-    "streaming_with_pauses": true,
-    "storage_critical_check": true,
-    "aggressive_ram_cleanup": true,
-    "aggressive_storage_cleanup": true
-  },
-  "config": {
-    "BATCH_SIZE": 16,
-    "datasets_to_exhaust": [
-      "dominguesm/restore-punctuation-ptbr-dataset",
-      "carolina-c4ai/corpus-carolina",
-      "CEIA-POSITIVO/ultrachat_br_clustred_balanced_v1",
-      "dominguesm/Canarim-Instruct-PTBR-Dataset"
-    ],
-    "SAMPLES_PER_DATASET_PHASE": 50,
-    "N_PHASES": 2,
-    "TOTAL_SAMPLES": 400,
-    "EPOCHS": 2,
-    "SOM_GRID": [
-      6,
-      6,
-      6,
-      4
-    ],
-    "n_neurons": 864,
-    "HIDDEN_DIM": 1024,
-    "VOCAB_SIZE": 16384,
-    "MAX_SEQ_LEN": 8,
-    "T_max": 10000,
-    "N_start": 10,
-    "lambda_ewc": 0.02,
-    "MTP_K": 6,
-    "MTP_ENTROPY_BETA": 0.01,
-    "MTP_ACTIVE_IN_VAL": true,
-    "VQVAE2_active": true,
-    "reasoning_engine_active": true,
-    "smoothquant_w8a8_active": true,
-    "INTER_BATCH_PAUSE_S": 0.3,
-    "INTER_DATASET_PAUSE_S": 1.5,
-    "STORAGE_CRITICAL_PCT": 90
-  },
-  "xeon_status": {
-    "version": "V6",
-    "physical_cores": 2,
-    "env": {
-      "MKL_ENABLE_INSTRUCTIONS": "AVX512",
-      "MKL_NUM_THREADS": "2",
-      "OMP_NUM_THREADS": "2",
-      "MKL_DYNAMIC": "FALSE",
-      "DNNL_PRIMITIVE_CACHE_CAPACITY": "1024",
-      "ONEDNN_MAX_CPU_ISA": "AMX_INT8",
-      "KMP_AFFINITY": "granularity=fine,compact,1,0",
-      "KMP_BLOCKTIME": "1"
-    },
-    "avx512": {
-      "supported": true,
-      "desc": "AVX512_VNNI (full INT8 acceleration)"
-    },
-    "amx": {
-      "supported": true,
-      "desc": "AMX (tile + int8 + bf16) — full AMX acceleration"
-    },
-    "ipex_available": false,
-    "init_done": true
-  },
-  "fp16_benchmark": {
-    "best_time_ms": 91.99277100015024,
-    "avg_time_ms": 93.86917700066988,
-    "best_tflops": 1.3914136796661005,
-    "avg_tflops": 1.3635998960456057,
-    "matrix_size": 4000.0
-  },
-  "training": {
-    "duration_s": 187.6828134059906,
-    "n_steps": 64,
-    "n_epochs": 2,
-    "n_phases": 2,
-    "samples_per_dataset_actual": {
-      "dominguesm/restore-punctuation-ptbr-dataset": 200,
-      "carolina-c4ai/corpus-carolina": 200,
-      "CEIA-POSITIVO/ultrachat_br_clustred_balanced_v1": 200,
-      "dominguesm/Canarim-Instruct-PTBR-Dataset": 200
-    },
-    "total_samples_processed": 800,
-    "streaming_failures": {
-      "dominguesm/restore-punctuation-ptbr-dataset": 0,
-      "carolina-c4ai/corpus-carolina": 0,
-      "CEIA-POSITIVO/ultrachat_br_clustred_balanced_v1": 0,
-      "dominguesm/Canarim-Instruct-PTBR-Dataset": 0
-    },
-    "storage_critical_stopped": false,
-    "inter_batch_pause_s": 0.3,
-    "inter_dataset_pause_s": 1.5
-  },
-  "summary": {
-    "n_steps": 64,
-    "final_loss": -1.0,
-    "mean_loss": -0.6892012953115315,
-    "min_loss": -1.0,
-    "max_loss": 5.0,
-    "final_acc": 1.0,
-    "mean_acc": 0.8977011614304813,
-    "sigma_start": 1.3846745195799537,
-    "sigma_end": 0.1596877565688792,
-    "alpha_start": 0.09607894391523232,
-    "alpha_end": 0.032627979462303947,
-    "vqvae2_mean_total_loss": 0.12848817340789304,
-    "vqvae2_final_total_loss": 0.12618379294872284,
-    "w8a8_mean_relative_error": 0.08495925303370692,
-    "w8a8_final_relative_error": 0.03702241876350528,
-    "rss_max_mb": 1757.76953125,
-    "rss_final_mb": 1757.76953125,
-    "disk_max_pct": 14.36071342865266,
-    "disk_final_pct": 14.36071342865266,
-    "n_alerts": 66,
-    "alerts": [
-      {
-        "type": "3.3_loss_vanishing",
-        "step": 1,
-        "value": -1.0
-      },
-      {
-        "type": "3.3_loss_vanishing",
-        "step": 2,
-        "value": -0.9682458365518543
-      },
-      {
-        "type": "3.3_loss_vanishing",
-        "step": 3,
-        "value": -1.0
-      },
-      {
-        "type": "3.3_loss_vanishing",
-        "step": 4,
-        "value": -0.9354143466934853
-      },
-      {
-        "type": "3.3_loss_vanishing",
-        "step": 5,
-        "value": -1.0
-      },
-      {
-        "type": "3.3_loss_vanishing",
-        "step": 6,
-        "value": -0.9354143466934853
-      },
-      {
-        "type": "3.3_loss_vanishing",
-        "step": 7,
-        "value": -1.0
-      },
-      {
-        "type": "3.3_loss_vanishing",
-        "step": 8,
-        "value": -0.9682458365518543
-      },
-      {
-        "type": "3.3_loss_vanishing",
-        "step": 9,
-        "value": -1.0
-      },
-      {
-        "type": "3.3_loss_vanishing",
-        "step": 10,
-        "value": -0.9354143466934853
-      },
-      {
-        "type": "3.3_loss_vanishing",
-        "step": 11,
-        "value": -1.0
-      },
-      {
-        "type": "3.1_loss_spike",
-        "step": 12,
-        "delta": 6.0
-      },
-      {
-        "type": "3.1_loss_spike",
-        "step": 13,
-        "delta": 6.0
-      },
-      {
-        "type": "3.3_loss_vanishing",
-        "step": 13,
-        "value": -1.0
-      },
-      {
-        "type": "3.3_loss_vanishing",
-        "step": 14,
-        "value": -1.0
-      },
-      {
-        "type": "3.3_loss_vanishing",
-        "step": 15,
-        "value": -1.0
-      },
-      {
-        "type": "3.3_loss_vanishing",
-        "step": 16,
-        "value": -0.747545001596402
-      },
-      {
-        "type": "3.3_loss_vanishing",
-        "step": 17,
-        "value": -1.0
-      },
-      {
-        "type": "3.3_loss_vanishing",
-        "step": 18,
-        "value": -0.9682458365518543
-      },
-      {
-        "type": "3.3_loss_vanishing",
-        "step": 19,
-        "value": -1.0
-      },
-      {
-        "type": "3.3_loss_vanishing",
-        "step": 20,
-        "value": -0.9354143466934853
-      },
-      {
-        "type": "3.3_loss_vanishing",
-        "step": 21,
-        "value": -1.0
-      },
-      {
-        "type": "3.3_loss_vanishing",
-        "step": 22,
-        "value": -0.9354143466934853
-      },
-      {
-        "type": "3.3_loss_vanishing",
-        "step": 23,
-        "value": -1.0
-      },
-      {
-        "type": "3.3_loss_vanishing",
-        "step": 24,
-        "value": -0.9682458365518543
-      },
-      {
-        "type": "3.3_loss_vanishing",
-        "step": 25,
-        "value": -1.0
-      },
-      {
-        "type": "3.3_loss_vanishing",
-        "step": 26,
-        "value": -0.9354143466934853
-      },
-      {
-        "type": "3.3_loss_vanishing",
-        "step": 27,
-        "value": -1.0
-      },
-      {
-        "type": "3.3_loss_vanishing",
-        "step": 28,
-        "value": -1.0
-      },
-      {
-        "type": "3.3_loss_vanishing",
-        "step": 29,
-        "value": -1.0
-      }
-    ],
-    "kohonen_final": {
-      "sigma_t": 0.1596877565688792,
-      "alpha_t": 0.032627979462303947,
-      "t": 2240,
-      "n_neurons": 864,
-      "fisher_w_mean": 0.0,
-      "fisher_w_max": 0.0,
-      "fisher_accum_count": 0,
-      "has_ewc_reference": true,
-      "weights_norm": 278.0921325683594,
-      "weights_w_mean": -0.0078024608083069324
-    },
-    "hypothesis_final": {
-      "classifier_trained": true,
-      "punishment_count": 0,
-      "success_count": 1,
-      "training_ready": true,
-      "buffer_size": 18,
-      "required_new_samples": 10,
-      "histogram_max": 18,
-      "time_counter": 828
-    },
-    "mtp_final": {
-      "active": false,
-      "mtp_total_loss": 0.0,
-      "mtp_weighted_loss": 0.0,
-      "mtp_entropy_reg": 0.0,
-      "mtp_alphas": [],
-      "mtp_per_k_loss": [],
-      "active_in_val": true,
-      "entropy_beta": 0.01,
-      "K": 6
-    },
-    "vqvae2_final": {
-      "active": true,
-      "n_calls": 28,
-      "latest_total_loss": 0.12618379294872284,
-      "latest_recon_loss": 0.12618273496627808,
-      "latest_vq_loss": 1.0548312729952158e-06,
-      "mean_total_loss": 0.13015546925641872
-    },
-    "w8a8_final": {
-      "active": true,
-      "n_calls": 28,
-      "latest_relative_error": 0.03702241876350528,
-      "latest_compression_ratio": 4.0,
-      "latest_preserves_topology": true,
-      "mean_relative_error": 0.0869346434271494,
-      "all_preserve_topology": false
-    },
-    "reasoning_final": {
-      "active": true,
-      "n_history": 0,
-      "n_steps_last": 0,
-      "phases_used": [],
-      "tool_coordinator_workers_active": true
-    },
-    "ewc_final": {
-      "active": true,
-      "ewc_eval_mode_penalty": true,
-      "w8a8_dequant_active": true,
-      "fisher_w_mean": 0.0,
-      "fisher_w_max": 0.0,
-      "fisher_accum_count": 0
-    }
-  },
-  "kohonen_final": {
-    "t": 2240,
-    "sigma_t": 0.1596877565688792,
-    "alpha_t": 0.032627979462303947,
-    "sigma0": 1.5,
-    "alpha0": 0.1,
-    "lambda_ewc": 0.02,
-    "grid_shape": [
-      6,
-      6,
-      6,
-      4
-    ],
-    "n_neurons": 864,
-    "has_ewc_reference": true,
-    "fisher_w_mean": 0.0,
-    "fisher_w_max": 0.0,
-    "fisher_accum_count": 0,
-    "weights_norm": 278.0921325683594,
-    "weights_w_mean": -0.0078024608083069324
-  },
-  "kls_state": {
-    "som": {
-      "t": 2240,
-      "sigma_t": 0.1596877565688792,
-      "alpha_t": 0.032627979462303947,
-      "sigma0": 1.5,
-      "alpha0": 0.1,
-      "lambda_ewc": 0.02,
-      "grid_shape": [
-        6,
-        6,
-        6,
-        4
-      ],
-      "n_neurons": 864,
-      "has_ewc_reference": true,
-      "fisher_w_mean": 0.0,
-      "fisher_w_max": 0.0,
-      "fisher_accum_count": 0,
-      "weights_norm": 278.0921325683594,
-      "weights_w_mean": -0.0078024608083069324
-    },
-    "kls": {
-      "time_counter": 842,
-      "T_max": 10000,
-      "buffer_size": 18,
-      "training_ready": true,
-      "punishment_count": 0,
-      "success_count": 1,
-      "classifier_trained": true,
-      "histogram_size": 1,
-      "histogram_max": 18,
-      "N_start": 10,
-      "dim_choice": "y",
-      "som_neuron_count": 864,
-      "required_new_samples": 10,
-      "has_classifier": true,
-      "vocab_size": 16384,
-      "hidden_dim": 1024,
-      "seq_len": 8,
-      "enable_vqvae2": true,
-      "enable_reasoning": true,
-      "enable_w8a8": true,
-      "vqvae2_n_calls": 28,
-      "w8a8_n_calls": 28,
-      "tool_coordinator_workers_active": true
-    },
-    "vqvae2": {
-      "active": true,
-      "n_calls": 28,
-      "latest": {
-        "vq_loss": 1.0548312729952158e-06,
-        "recon_loss": 0.12618273496627808,
-        "total_loss": 0.12618379294872284,
-        "n_used_top": 64,
-        "n_used_bot": 123,
-        "usage_ratio_top": 1.0,
-        "usage_ratio_bot": 0.9609375,
-        "codebook_ppl_top": 14.672064691054654,
-        "codebook_ppl_bot": 14.672064691054654,
-        "n_restarted_top": 0,
-        "n_restarted_bot": 0,
-        "goose_temp": 0.5,
-        "active": true
-      },
-      "mean_total_loss": 0.13015546925641872,
-      "mean_recon_loss": 0.12062281556427479,
-      "n_nan_skipped": 1
-    },
-    "reasoning": {
-      "active": true,
-      "stats": {
-        "n_steps": 8,
-        "n_history": 7,
-        "tools": [
-          "som_query",
-          "buffer_stats"
-        ],
-        "tool_stats": {
-          "som_query": {
-            "n_calls": 35,
-            "n_success": 35,
-            "n_failures": 0,
-            "total_time_s": 0.12173104286193848,
-            "avg_time_s": 0.003478029796055385,
-            "cache_size": 0,
-            "instructions_count": 0,
-            "error_count": 0,
-            "success_rate": 1.0,
-            "circuit_breaker": {
-              "state": "closed",
-              "failure_count": 0,
-              "success_count": 0,
-              "failure_threshold": 5,
-              "recovery_timeout_s": 30.0
-            }
-          },
-          "buffer_stats": {
-            "n_calls": 28,
-            "n_success": 28,
-            "n_failures": 0,
-            "total_time_s": 0.02679157257080078,
-            "avg_time_s": 0.0009568418775285993,
-            "cache_size": 0,
-            "instructions_count": 0,
-            "error_count": 0,
-            "success_rate": 1.0,
-            "circuit_breaker": {
-              "state": "closed",
-              "failure_count": 0,
-              "success_count": 0,
-              "failure_threshold": 5,
-              "recovery_timeout_s": 30.0
-            }
-          }
-        },
-        "phases_used": [
-          "planning",
-          "monitoring",
-          "executing",
-          "predicting",
-          "adjusting",
-          "decomposing",
-          "answering",
-          "thinking"
-        ]
-      },
-      "n_history": 7
-    },
-    "w8a8": {
-      "active": true,
-      "n_calls": 28,
-      "latest": {
-        "active": true,
-        "n_neurons": 864,
-        "n_features": 4,
-        "n_calibration_samples": 16,
-        "smooth_scale_mean": 0.042823534458875656,
-        "smooth_scale_std": 0.08385249227285385,
-        "smooth_scale_min": 0.0008972898940555751,
-        "smooth_scale_max": 0.16860227286815643,
-        "alpha": 0.5,
-        "n_bits": 8,
-        "compression_ratio": 4.0,
-        "weights_norm_original": 268.19549560546875,
-        "weights_norm_dequant": 278.0921325683594,
-        "absolute_error": 9.929245948791504,
-        "relative_error": 0.03702241876350528,
-        "preserves_topology": true,
-        "w8a8_int8_range": [
-          -123,
-          127
-        ],
-        "elapsed_ms": 0.6420612335205078
-      },
-      "mean_relative_error": 0.0869346434271494,
-      "max_relative_error": 0.18615433008559695,
-      "all_preserve_topology": false
-    }
-  },
-  "verification": {
-    "checks": {
-      "kls_instantiation": {
-        "status": "PASS",
-        "details": "vqvae2=True, reasoning=True, w8a8=True, workers_active=True"
-      },
-      "find_bmu_no_premature_return": {
-        "status": "PASS",
-        "details": "bmu=(0, 2, 2, 0) valid"
-      },
-      "activate_hypothesis_detach": {
-        "status": "PASS",
-        "details": "classifier_trained=True"
-      },
-      "pgvector_lookup_removed": {
-        "status": "PASS"
-      },
-      "kohonen_refactored_removed": {
-        "status": "PASS",
-        "details": "old_folder=False, new_file=True"
-      },
-      "trainer_removed": {
-        "status": "PASS"
-      },
-      "vqvae2_valid_output": {
-        "status": "PASS",
-        "details": "n_calls=1, total_loss=0.0972"
-      },
-      "reasoning_engine_tags": {
-        "status": "PASS",
-        "details": "reasoning length=942 chars"
-      },
-      "smoothquant_w8a8_som_weights": {
-        "status": "PASS",
-        "details": "relative_error=0.011594, compression_ratio=4.0x, n_neurons=54"
-      },
-      "tool_coordinator_workers_reactivated": {
-        "status": "PASS",
-        "details": "n_workers=4, n_tools=2, n_results=2, elapsed_ms=2.67"
-      }
-    },
-    "n_pass": 10,
-    "n_fail": 0,
-    "all_pass": true
-  },
-  "ewc_w8a8_benchmark_summary": {
-    "dequant_preserves_accuracy": true,
-    "dequant_relative_error": 0.00027968583245277626,
-    "int8_relative_error": 145504.6191166922,
-    "dequant_overhead_ms": 0.08262872695922852,
-    "dequant_overhead_pct": 181.2225475841874,
-    "conclusion": "V6.5-final EWC+W8A8 eval com dequant: erro dequant=0.000280 (< 0.1 = OK), erro int8 direto=145504.619117."
-  },
-  "reasoning_eval_summary": {
-    "n_with_answer": 7,
-    "n_with_think": 7,
-    "answer_rate": 1.0,
-    "think_rate": 1.0,
-    "avg_reasoning_length": 1026.5714285714287,
-    "reasoning_engine_active": true,
-    "reasoning_engine_n_history": 7
-  },
-  "w8a8_compression_summary": {
-    "active": true,
-    "n_calls": 28,
-    "latest": {
-      "active": true,
-      "n_neurons": 864,
-      "n_features": 4,
-      "n_calibration_samples": 16,
-      "smooth_scale_mean": 0.042823534458875656,
-      "smooth_scale_std": 0.08385249227285385,
-      "smooth_scale_min": 0.0008972898940555751,
-      "smooth_scale_max": 0.16860227286815643,
-      "alpha": 0.5,
-      "n_bits": 8,
-      "compression_ratio": 4.0,
-      "weights_norm_original": 268.19549560546875,
-      "weights_norm_dequant": 278.0921325683594,
-      "absolute_error": 9.929245948791504,
-      "relative_error": 0.03702241876350528,
-      "preserves_topology": true,
-      "w8a8_int8_range": [
-        -123,
-        127
-      ],
-      "elapsed_ms": 0.6420612335205078
-    },
-    "mean_relative_error": 0.0869346434271494,
-    "max_relative_error": 0.18615433008559695,
-    "all_preserve_topology": false
-  },
-  "module_analysis_summary": {
-    "n_removed_modules": 9,
-    "n_removed_folders": 1,
-    "n_active_modules": 21
-  },
-  "script_activity_summary": {
-    "n_scripts": 5,
-    "active_v65_final": 1
-  },
-  "math_analysis": {
-    "text_to_4d": "SVD: M @ V[:3].T -> centroid 3D + w = time_step/T_max (LINEAR)",
-    "bmu_distance": "||W - x||^2 (L2 squared in R^4)",
-    "neighborhood": "Lambda(d, sigma) = exp(-d^2 / (2*sigma^2))",
-    "weight_update": "dW = alpha * Lambda * (x - W)",
-    "sigma_decay": "sigma_t = sigma0 * exp(-t/1000)",
-    "alpha_decay": "alpha_t = alpha0 * exp(-t/2000)",
-    "ewc_only_dim4": "penalty = lambda * F * (W_w - W*_w)",
-    "vqvae2_loss": "L = recon_loss + vq_loss (commitment top + bottom + diversity)",
-    "smoothquant_w8a8": "s_j = max|X_j|^alpha / max|W_j|^(1-alpha); W_smooth = W * diag(s); INT8 quant; dequant = W_int8 * scale / diag(s)",
-    "mtp_loss": "L = sum_k(alpha_k * L_k) - beta * H(alpha), K=6 (V6.5-final)",
-    "ewc_w8a8_dequant": "W_float = (W_int8 * scale) / smooth_scale; EWC uses W_float for (p-w*)^2"
-  },
-  "datasets_used": [
-    "dominguesm/restore-punctuation-ptbr-dataset",
-    "carolina-c4ai/corpus-carolina",
-    "CEIA-POSITIVO/ultrachat_br_clustred_balanced_v1",
-    "dominguesm/Canarim-Instruct-PTBR-Dataset"
-  ],
-  "storage_cleanup_pre": {
-    "n_files_removed": 0,
-    "bytes_freed": 0,
-    "mb_freed": 0.0,
-    "files_removed_sample": []
-  }
-}