File size: 1,195 Bytes
69da4f2
 
 
 
 
 
 
522e4d5
 
 
 
 
 
69da4f2
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
{
  "hf_repo": "conceptmodels/fmac",
  "upload_milestones": [
    0.5,
    0.75,
    1.0
  ],
  "pretrain_dataset": "HuggingFaceTB/smollm-corpus",
  "pretrain_config": "cosmopedia-v2",
  "sft_dataset": "HuggingFaceTB/smoltalk",
  "sft_config": "everyday-conversations",
  "gsm_dataset": "openai/gsm8k",
  "replay_frac": 0.4,
  "vocab": 50304,
  "real_vocab": 50257,
  "eos_id": 50256,
  "d": 384,
  "layers": 6,
  "heads": 6,
  "L": 256,
  "max_pos": 1024,
  "max_ctx": 512,
  "window": 128,
  "base_chunk": 16,
  "pyr_ratio": 8,
  "pyr_recent": 4,
  "max_levels": 5,
  "shared_h": 384,
  "nodes": 8,
  "topk": 2,
  "node_h": 384,
  "cap_factor": 1.75,
  "router_rank": 32,
  "router_metric": "cosine",
  "bias_update_rate": 0.001,
  "lam_bal": 0.001,
  "lam_z": 0.0001,
  "lam_null": 0.25,
  "null_target": 0.2,
  "gcv_alpha": 0.5,
  "gcv_decay_bias": 4.0,
  "total_steps": 4000,
  "stage_a_frac": 0.75,
  "warmup_steps": 200,
  "cooldown_frac": 0.15,
  "micro_bs": 16,
  "accum": 4,
  "lr_muon": 0.005,
  "lr_adam": 0.0006,
  "min_lr_frac": 0.05,
  "wd": 0.05,
  "clip": 1.0,
  "prune_p": 0.1,
  "compile": true,
  "ckpt_dir": "/content/checkpoints",
  "log_every": 25,
  "interactive": true
}