ConceptModels commited on
Commit
69da4f2
·
verified ·
1 Parent(s): d233af0

Upload config.json with huggingface_hub

Browse files
Files changed (1) hide show
  1. config.json +57 -0
config.json ADDED
@@ -0,0 +1,57 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "hf_repo": "conceptmodels/fmac",
3
+ "upload_milestones": [
4
+ 0.5,
5
+ 0.75,
6
+ 1.0
7
+ ],
8
+ "vocab": 50304,
9
+ "real_vocab": 50257,
10
+ "eos_id": 50256,
11
+ "d": 384,
12
+ "layers": 6,
13
+ "heads": 6,
14
+ "L": 256,
15
+ "max_pos": 1024,
16
+ "max_ctx": 512,
17
+ "window": 128,
18
+ "base_chunk": 16,
19
+ "pyr_ratio": 8,
20
+ "pyr_recent": 4,
21
+ "max_levels": 5,
22
+ "shared_h": 384,
23
+ "nodes": 8,
24
+ "topk": 2,
25
+ "node_h": 384,
26
+ "cap_factor": 1.75,
27
+ "router_rank": 32,
28
+ "router_metric": "cosine",
29
+ "bias_update_rate": 0.001,
30
+ "lam_bal": 0.001,
31
+ "lam_z": 0.0001,
32
+ "lam_null": 0.25,
33
+ "null_target": 0.2,
34
+ "gcv_alpha": 0.5,
35
+ "gcv_decay_bias": 4.0,
36
+ "total_steps": 4000,
37
+ "stage_a_frac": 0.75,
38
+ "warmup_steps": 200,
39
+ "cooldown_frac": 0.15,
40
+ "micro_bs": 16,
41
+ "accum": 4,
42
+ "lr_muon": 0.005,
43
+ "lr_adam": 0.0006,
44
+ "min_lr_frac": 0.05,
45
+ "wd": 0.05,
46
+ "clip": 1.0,
47
+ "prune_p": 0.1,
48
+ "pretrain_dataset": "HuggingFaceFW/fineweb-edu",
49
+ "pretrain_config": "sample-10BT",
50
+ "sft_dataset": "yahma/alpaca-cleaned",
51
+ "gsm_dataset": "openai/gsm8k",
52
+ "replay_frac": 0.4,
53
+ "compile": true,
54
+ "ckpt_dir": "/content/checkpoints",
55
+ "log_every": 25,
56
+ "interactive": true
57
+ }