fmac-lobotomy / config.json
ConceptModels's picture
Upload config.json with huggingface_hub
522e4d5 verified
Raw History Blame Contribute Delete
1.2 kB
{
"hf_repo": "conceptmodels/fmac",
"upload_milestones": [
0.5,
0.75,
1.0
],
"pretrain_dataset": "HuggingFaceTB/smollm-corpus",
"pretrain_config": "cosmopedia-v2",
"sft_dataset": "HuggingFaceTB/smoltalk",
"sft_config": "everyday-conversations",
"gsm_dataset": "openai/gsm8k",
"replay_frac": 0.4,
"vocab": 50304,
"real_vocab": 50257,
"eos_id": 50256,
"d": 384,
"layers": 6,
"heads": 6,
"L": 256,
"max_pos": 1024,
"max_ctx": 512,
"window": 128,
"base_chunk": 16,
"pyr_ratio": 8,
"pyr_recent": 4,
"max_levels": 5,
"shared_h": 384,
"nodes": 8,
"topk": 2,
"node_h": 384,
"cap_factor": 1.75,
"router_rank": 32,
"router_metric": "cosine",
"bias_update_rate": 0.001,
"lam_bal": 0.001,
"lam_z": 0.0001,
"lam_null": 0.25,
"null_target": 0.2,
"gcv_alpha": 0.5,
"gcv_decay_bias": 4.0,
"total_steps": 4000,
"stage_a_frac": 0.75,
"warmup_steps": 200,
"cooldown_frac": 0.15,
"micro_bs": 16,
"accum": 4,
"lr_muon": 0.005,
"lr_adam": 0.0006,
"min_lr_frac": 0.05,
"wd": 0.05,
"clip": 1.0,
"prune_p": 0.1,
"compile": true,
"ckpt_dir": "/content/checkpoints",
"log_every": 25,
"interactive": true
}