AdminHugging commited on
Commit
022ee4d
·
verified ·
1 Parent(s): a6d3990

Upload logs/config_used.yaml with huggingface_hub

Browse files
Files changed (1) hide show
  1. logs/config_used.yaml +61 -0
logs/config_used.yaml ADDED
@@ -0,0 +1,61 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ root: /kaggle/working/ppai
2
+ seed: 1337
3
+ model:
4
+ base_id: AuroraAI-Research/Aurora2-400M-A20M
5
+ subfolder: bf16
6
+ base_dir: /kaggle/working/ppai/model/base
7
+ tokenizer:
8
+ extend: false
9
+ new_tokens: 16000
10
+ word_share: 0.6
11
+ sample_chars: 20000000
12
+ min_word_freq: 40
13
+ min_word_len: 3
14
+ corpus: ru_wiki
15
+ corpus_file: null
16
+ data:
17
+ total_tokens: 500000000
18
+ seq_len: 1024
19
+ shard_tokens: 50000000
20
+ val_tokens_per_source: 200000
21
+ seed: 1337
22
+ dedup_prefix_chars: 200
23
+ max_scan_per_source: 400000
24
+ use_weights: true
25
+ sources:
26
+ ru_wiki: 0.12
27
+ ru_web: 0.22
28
+ ru_c4: 0.06
29
+ math_numina: 0.12
30
+ math_open: 0.08
31
+ math_meta: 0.05
32
+ code_python: 0.34
33
+ code_csharp: 0.03
34
+ code_rust: 0.03
35
+ en_web: 0.2
36
+ optim:
37
+ lr: 0.0006
38
+ min_lr_ratio: 0.05
39
+ warmup_steps: 500
40
+ weight_decay: 0.1
41
+ beta1: 0.9
42
+ beta2: 0.95
43
+ grad_clip: 1.0
44
+ micro_batch: 12
45
+ grad_accum: 3
46
+ dtype: auto
47
+ params_dtype: fp32
48
+ train:
49
+ train_mode: full
50
+ freeze_embeddings: false
51
+ max_steps: 6780
52
+ save_every: 250
53
+ eval_every: 500
54
+ log_every: 25
55
+ max_hours: 10.5
56
+ resume: true
57
+ hub_repo: AdminHugging/PP-Hydrogen
58
+ hub_private: true
59
+ push_every: 1000
60
+ keep_step_ckpts: 1
61
+ target_tokens: 500000000