AdminHugging commited on
Commit
089c19d
·
verified ·
1 Parent(s): fd2686d

Upload logs/config_used.yaml with huggingface_hub

Browse files
Files changed (1) hide show
  1. logs/config_used.yaml +63 -61
logs/config_used.yaml CHANGED
@@ -1,61 +1,63 @@
1
- root: /kaggle/working/ppai
2
- seed: 1337
3
- model:
4
- base_id: PPAI/PP-H
5
- subfolder: bf16
6
- base_dir: /kaggle/working/ppai/model/base
7
- tokenizer:
8
- extend: false
9
- new_tokens: 16000
10
- word_share: 0.6
11
- sample_chars: 20000000
12
- min_word_freq: 40
13
- min_word_len: 3
14
- corpus: ru_wiki
15
- corpus_file: null
16
- data:
17
- total_tokens: 500000000
18
- seq_len: 1024
19
- shard_tokens: 50000000
20
- val_tokens_per_source: 200000
21
- seed: 1337
22
- dedup_prefix_chars: 200
23
- max_scan_per_source: 400000
24
- use_weights: true
25
- sources:
26
- ru_wiki: 0.12
27
- ru_web: 0.22
28
- ru_c4: 0.06
29
- math_numina: 0.12
30
- math_open: 0.08
31
- math_meta: 0.05
32
- code_python: 0.34
33
- code_csharp: 0.03
34
- code_rust: 0.03
35
- en_web: 0.2
36
- optim:
37
- lr: 0.0006
38
- min_lr_ratio: 0.05
39
- warmup_steps: 500
40
- weight_decay: 0.1
41
- beta1: 0.9
42
- beta2: 0.95
43
- grad_clip: 1.0
44
- micro_batch: 12
45
- grad_accum: 3
46
- dtype: auto
47
- params_dtype: fp32
48
- train:
49
- train_mode: full
50
- freeze_embeddings: false
51
- max_steps: 6780
52
- save_every: 250
53
- eval_every: 500
54
- log_every: 25
55
- max_hours: 10.5
56
- resume: true
57
- hub_repo: AdminHugging/PP-Hydrogen
58
- hub_private: true
59
- push_every: 1000
60
- keep_step_ckpts: 1
61
- target_tokens: 500000000
 
 
 
1
+ root: /kaggle/working/ppai
2
+ seed: 1337
3
+ model:
4
+ base_id: PPAI/PP-H
5
+ subfolder: bf16
6
+ base_dir: /kaggle/working/ppai/model/base
7
+ tokenizer:
8
+ extend: false
9
+ new_tokens: 16000
10
+ word_share: 0.6
11
+ sample_chars: 20000000
12
+ min_word_freq: 40
13
+ min_word_len: 3
14
+ corpus: ru_wiki
15
+ corpus_file: null
16
+ data:
17
+ total_tokens: 1000000000
18
+ seq_len: 1024
19
+ shard_tokens: 50000000
20
+ val_tokens_per_source: 200000
21
+ seed: 1337
22
+ dedup_prefix_chars: 200
23
+ max_scan_per_source: 400000
24
+ use_weights: true
25
+ sources:
26
+ ru_wiki: 0.12
27
+ ru_web: 0.22
28
+ ru_c4: 0.06
29
+ math_numina: 0.12
30
+ math_open: 0.08
31
+ math_meta: 0.05
32
+ code_python: 0.34
33
+ code_csharp: 0.03
34
+ code_rust: 0.03
35
+ en_web: 0.2
36
+ optim:
37
+ lr: 0.0006
38
+ min_lr_ratio: 0.05
39
+ warmup_steps: 500
40
+ weight_decay: 0.1
41
+ beta1: 0.9
42
+ beta2: 0.95
43
+ grad_clip: 1.0
44
+ micro_batch: 12
45
+ grad_accum: 3
46
+ dtype: auto
47
+ device: auto
48
+ params_dtype: fp32
49
+ train:
50
+ train_mode: full
51
+ freeze_embeddings: false
52
+ max_steps: 40690
53
+ save_every: 250
54
+ eval_every: 500
55
+ log_every: 25
56
+ max_hours: 10.5
57
+ resume: true
58
+ hub_repo: AdminHugging/PP-Hydrogen
59
+ hub_private: true
60
+ push_every: 1000
61
+ keep_step_ckpts: 1
62
+ target_tokens: 1500000000
63
+ require_resume: true