| { |
| "architecture": "BET_Fog_mHC_Time", |
| "architecture_shape_sha256": "145598b10b7aee6d350ccd16082121003ee4a666eb3fbaf941dd269997fff793", |
| "completed_steps": 65000, |
| "data_contract": { |
| "data_seed": 2026, |
| "dataset_config": "sample-10BT", |
| "dataset_name": "HuggingFaceFW/fineweb-edu", |
| "train_fingerprint": "72787368fca783a02a95856aeac9c2dcfc6a076d88822379a751c79d3fffa9b6", |
| "train_ids": 512000000, |
| "val_fingerprint": "2b1601a78453c8a35ac5ade77c9aa2417fd530d83fd75d4f22b5e7c1c83b90fe", |
| "val_ids": 8000000 |
| }, |
| "model_config": { |
| "head_dim": 64, |
| "hidden_size": 384, |
| "intermediate_size": 960, |
| "max_seq_len": 512, |
| "num_heads": 12, |
| "num_kv_heads": 1, |
| "num_layers": 6, |
| "rms_eps": 1e-06, |
| "rope_theta": 1000000.0, |
| "vocab_size": 259 |
| }, |
| "parameter_count": 10881809, |
| "source_checkpoint": "step_065000.pt", |
| "training_contract": { |
| "adam_betas": [ |
| 0.9, |
| 0.95 |
| ], |
| "adam_eps": 1e-08, |
| "arch_shape_sha256": "145598b10b7aee6d350ccd16082121003ee4a666eb3fbaf941dd269997fff793", |
| "architecture": "BET_Fog_mHC_Time", |
| "base_lr": 0.0006, |
| "data_seed": 2026, |
| "elastic_long_loops": 6, |
| "elastic_short_loops": 3, |
| "global_tokens_per_update": 4096, |
| "grad_accum_steps": 1, |
| "grad_clip": 1.0, |
| "grad_scaler_backoff_factor": 0.5, |
| "grad_scaler_growth_factor": 2.0, |
| "grad_scaler_growth_interval": 2000, |
| "grad_scaler_init_scale": 1024.0, |
| "micro_batch_per_gpu": 4, |
| "min_lr": 6e-05, |
| "param_count": 10881809, |
| "schedule_total_steps": 100000, |
| "seq_len": 512, |
| "short_loss_weight": 0.1, |
| "warmup_steps": 100, |
| "weight_decay": 0.1, |
| "world_size": 2 |
| } |
| } |
|
|
