File size: 1,690 Bytes
9f696a5 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 | {
"architecture": "BET_Fog_mHC_Time",
"architecture_shape_sha256": "145598b10b7aee6d350ccd16082121003ee4a666eb3fbaf941dd269997fff793",
"completed_steps": 65000,
"data_contract": {
"data_seed": 2026,
"dataset_config": "sample-10BT",
"dataset_name": "HuggingFaceFW/fineweb-edu",
"train_fingerprint": "72787368fca783a02a95856aeac9c2dcfc6a076d88822379a751c79d3fffa9b6",
"train_ids": 512000000,
"val_fingerprint": "2b1601a78453c8a35ac5ade77c9aa2417fd530d83fd75d4f22b5e7c1c83b90fe",
"val_ids": 8000000
},
"model_config": {
"head_dim": 64,
"hidden_size": 384,
"intermediate_size": 960,
"max_seq_len": 512,
"num_heads": 12,
"num_kv_heads": 1,
"num_layers": 6,
"rms_eps": 1e-06,
"rope_theta": 1000000.0,
"vocab_size": 259
},
"parameter_count": 10881809,
"source_checkpoint": "step_065000.pt",
"training_contract": {
"adam_betas": [
0.9,
0.95
],
"adam_eps": 1e-08,
"arch_shape_sha256": "145598b10b7aee6d350ccd16082121003ee4a666eb3fbaf941dd269997fff793",
"architecture": "BET_Fog_mHC_Time",
"base_lr": 0.0006,
"data_seed": 2026,
"elastic_long_loops": 6,
"elastic_short_loops": 3,
"global_tokens_per_update": 4096,
"grad_accum_steps": 1,
"grad_clip": 1.0,
"grad_scaler_backoff_factor": 0.5,
"grad_scaler_growth_factor": 2.0,
"grad_scaler_growth_interval": 2000,
"grad_scaler_init_scale": 1024.0,
"micro_batch_per_gpu": 4,
"min_lr": 6e-05,
"param_count": 10881809,
"schedule_total_steps": 100000,
"seq_len": 512,
"short_loss_weight": 0.1,
"warmup_steps": 100,
"weight_decay": 0.1,
"world_size": 2
}
}
|