File size: 1,195 Bytes
69da4f2 522e4d5 69da4f2 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 | {
"hf_repo": "conceptmodels/fmac",
"upload_milestones": [
0.5,
0.75,
1.0
],
"pretrain_dataset": "HuggingFaceTB/smollm-corpus",
"pretrain_config": "cosmopedia-v2",
"sft_dataset": "HuggingFaceTB/smoltalk",
"sft_config": "everyday-conversations",
"gsm_dataset": "openai/gsm8k",
"replay_frac": 0.4,
"vocab": 50304,
"real_vocab": 50257,
"eos_id": 50256,
"d": 384,
"layers": 6,
"heads": 6,
"L": 256,
"max_pos": 1024,
"max_ctx": 512,
"window": 128,
"base_chunk": 16,
"pyr_ratio": 8,
"pyr_recent": 4,
"max_levels": 5,
"shared_h": 384,
"nodes": 8,
"topk": 2,
"node_h": 384,
"cap_factor": 1.75,
"router_rank": 32,
"router_metric": "cosine",
"bias_update_rate": 0.001,
"lam_bal": 0.001,
"lam_z": 0.0001,
"lam_null": 0.25,
"null_target": 0.2,
"gcv_alpha": 0.5,
"gcv_decay_bias": 4.0,
"total_steps": 4000,
"stage_a_frac": 0.75,
"warmup_steps": 200,
"cooldown_frac": 0.15,
"micro_bs": 16,
"accum": 4,
"lr_muon": 0.005,
"lr_adam": 0.0006,
"min_lr_frac": 0.05,
"wd": 0.05,
"clip": 1.0,
"prune_p": 0.1,
"compile": true,
"ckpt_dir": "/content/checkpoints",
"log_every": 25,
"interactive": true
} |