AdminHugging commited on
Commit
b30c578
·
verified ·
1 Parent(s): 27cfe0a

Upload ckpt/config.json with huggingface_hub

Browse files
Files changed (1) hide show
  1. ckpt/config.json +70 -68
ckpt/config.json CHANGED
@@ -1,69 +1,71 @@
1
- {
2
- "root": "/kaggle/working/ppai",
3
- "seed": 1337,
4
- "model": {
5
- "base_id": "PPAI/PP-H",
6
- "subfolder": "bf16",
7
- "base_dir": "/kaggle/working/ppai/model/base"
8
- },
9
- "tokenizer": {
10
- "extend": false,
11
- "new_tokens": 16000,
12
- "word_share": 0.6,
13
- "sample_chars": 20000000,
14
- "min_word_freq": 40,
15
- "min_word_len": 3,
16
- "corpus": "ru_wiki",
17
- "corpus_file": null
18
- },
19
- "data": {
20
- "total_tokens": 500000000,
21
- "seq_len": 1024,
22
- "shard_tokens": 50000000,
23
- "val_tokens_per_source": 200000,
24
- "seed": 1337,
25
- "dedup_prefix_chars": 200,
26
- "max_scan_per_source": 400000,
27
- "use_weights": true,
28
- "sources": {
29
- "ru_wiki": 0.12,
30
- "ru_web": 0.22,
31
- "ru_c4": 0.06,
32
- "math_numina": 0.12,
33
- "math_open": 0.08,
34
- "math_meta": 0.05,
35
- "code_python": 0.34,
36
- "code_csharp": 0.03,
37
- "code_rust": 0.03,
38
- "en_web": 0.2
39
- }
40
- },
41
- "optim": {
42
- "lr": 0.0006,
43
- "min_lr_ratio": 0.05,
44
- "warmup_steps": 500,
45
- "weight_decay": 0.1,
46
- "beta1": 0.9,
47
- "beta2": 0.95,
48
- "grad_clip": 1.0,
49
- "micro_batch": 12,
50
- "grad_accum": 3,
51
- "dtype": "auto",
52
- "params_dtype": "fp32"
53
- },
54
- "train": {
55
- "train_mode": "full",
56
- "freeze_embeddings": false,
57
- "max_steps": 6780,
58
- "save_every": 250,
59
- "eval_every": 500,
60
- "log_every": 25,
61
- "max_hours": 10.5,
62
- "resume": true,
63
- "hub_repo": "AdminHugging/PP-Hydrogen",
64
- "hub_private": true,
65
- "push_every": 1000,
66
- "keep_step_ckpts": 1,
67
- "target_tokens": 500000000
68
- }
 
 
69
  }
 
1
+ {
2
+ "root": "/kaggle/working/ppai",
3
+ "seed": 1337,
4
+ "model": {
5
+ "base_id": "PPAI/PP-H",
6
+ "subfolder": "bf16",
7
+ "base_dir": "/kaggle/working/ppai/model/base"
8
+ },
9
+ "tokenizer": {
10
+ "extend": false,
11
+ "new_tokens": 16000,
12
+ "word_share": 0.6,
13
+ "sample_chars": 20000000,
14
+ "min_word_freq": 40,
15
+ "min_word_len": 3,
16
+ "corpus": "ru_wiki",
17
+ "corpus_file": null
18
+ },
19
+ "data": {
20
+ "total_tokens": 1000000000,
21
+ "seq_len": 1024,
22
+ "shard_tokens": 50000000,
23
+ "val_tokens_per_source": 200000,
24
+ "seed": 1337,
25
+ "dedup_prefix_chars": 200,
26
+ "max_scan_per_source": 400000,
27
+ "use_weights": true,
28
+ "sources": {
29
+ "ru_wiki": 0.12,
30
+ "ru_web": 0.22,
31
+ "ru_c4": 0.06,
32
+ "math_numina": 0.12,
33
+ "math_open": 0.08,
34
+ "math_meta": 0.05,
35
+ "code_python": 0.34,
36
+ "code_csharp": 0.03,
37
+ "code_rust": 0.03,
38
+ "en_web": 0.2
39
+ }
40
+ },
41
+ "optim": {
42
+ "lr": 0.0006,
43
+ "min_lr_ratio": 0.05,
44
+ "warmup_steps": 500,
45
+ "weight_decay": 0.1,
46
+ "beta1": 0.9,
47
+ "beta2": 0.95,
48
+ "grad_clip": 1.0,
49
+ "micro_batch": 12,
50
+ "grad_accum": 3,
51
+ "dtype": "auto",
52
+ "device": "auto",
53
+ "params_dtype": "fp32"
54
+ },
55
+ "train": {
56
+ "train_mode": "full",
57
+ "freeze_embeddings": false,
58
+ "max_steps": 40690,
59
+ "save_every": 250,
60
+ "eval_every": 500,
61
+ "log_every": 25,
62
+ "max_hours": 10.5,
63
+ "resume": true,
64
+ "hub_repo": "AdminHugging/PP-Hydrogen",
65
+ "hub_private": true,
66
+ "push_every": 1000,
67
+ "keep_step_ckpts": 1,
68
+ "target_tokens": 1500000000,
69
+ "require_resume": true
70
+ }
71
  }