mmarone commited on
Commit
63416ae
·
verified ·
1 Parent(s): 10bbbcd

Delete config.yaml

Browse files
Files changed (1) hide show
  1. config.yaml +0 -116
config.yaml DELETED
@@ -1,116 +0,0 @@
1
- checkpoints:
2
- checkpoint_interval: 191
3
- checkpoints_path: /fsx/marcmarone/upcycle/artifacts/nanotron_checkpoints/upcycle-670m-c4-seed42
4
- checkpoints_path_is_shared_file_system: true
5
- load_lr_scheduler: true
6
- load_optimizer: true
7
- resume_checkpoint_path: /fsx/marcmarone/upcycle/artifacts/nanotron_checkpoints/upcycle-670m-c4-seed42/4966
8
- save_final_state: true
9
- save_initial_state: false
10
- data_stages:
11
- - data:
12
- dataset:
13
- dataset_overwrite_cache: false
14
- dataset_processing_batch_size: 1000
15
- dataset_processing_num_proc_per_process: 16
16
- dataset_shuffle: false
17
- hf_dataset_config_name: default
18
- hf_dataset_or_datasets: /fsx/marcmarone/upcycle/prepared/c4-megadocsrewritten-6.7b-retok
19
- hf_dataset_splits: train
20
- text_column_name: text
21
- num_loading_workers: 2
22
- seed: 42
23
- name: FineWeb Stage (epoch 1)
24
- sequence_length: 4096
25
- start_training_step: 1
26
- general:
27
- benchmark_csv_path: /fsx/marcmarone/upcycle/artifacts/nanotron_benchmarks/upcycle-670m-c4-seed42.csv
28
- consumed_train_samples: 1626368
29
- ignore_sanity_checks: true
30
- project: upcycle
31
- run: upcycle-670m-c4-seed42
32
- seed: 42
33
- step: 6353
34
- lighteval: null
35
- logging:
36
- iteration_step_info_interval: 1
37
- log_level: info
38
- log_level_replica: info
39
- metrics_logging: null
40
- model:
41
- ddp_bucket_cap_mb: 25
42
- dtype: bfloat16
43
- init_method:
44
- scaling_method: NUM_LAYERS
45
- std: 0.02
46
- make_vocab_size_divisible_by: 1
47
- model_config:
48
- _attn_implementation: flash_attention_2
49
- attention_bias: false
50
- bos_token_id: 1
51
- eos_token_id: 2
52
- hidden_act: silu
53
- hidden_size: 1024
54
- initializer_range: 0.02
55
- intermediate_size: 7168
56
- is_llama_config: true
57
- max_position_embeddings: 4096
58
- num_attention_heads: 16
59
- num_hidden_layers: 24
60
- num_key_value_heads: 8
61
- pad_token_id: 0
62
- pretraining_tp: 1
63
- rms_norm_eps: 1.0e-05
64
- rope_interleaved: false
65
- rope_scaling: null
66
- rope_theta: 10000.0
67
- tie_word_embeddings: false
68
- use_cache: true
69
- vocab_size: 32000
70
- z_loss_coefficient: 0.0001
71
- z_loss_enabled: false
72
- optimizer:
73
- accumulate_grad_in_fp32: true
74
- clip_grad: 1.0
75
- learning_rate_scheduler:
76
- learning_rate: 0.001
77
- lr_decay_starting_step: null
78
- lr_decay_steps: 6353
79
- lr_decay_style: cosine
80
- lr_warmup_steps: 256
81
- lr_warmup_style: linear
82
- min_decay_lr: 0.0001
83
- optimizer_factory:
84
- adam_beta1: 0.9
85
- adam_beta2: 0.95
86
- adam_eps: 1.0e-08
87
- name: adamW
88
- torch_adam_is_fused: true
89
- weight_decay: 1.6
90
- weight_decay_exclude_named_params: []
91
- zero_stage: 0
92
- parallelism:
93
- context_parallel_size: 1
94
- dp: 8
95
- expert_parallel_size: 1
96
- pp: 1
97
- pp_engine: 1f1b
98
- recompute_layer: false
99
- tp: 1
100
- tp_linear_async_communication: true
101
- tp_mode: REDUCE_SCATTER
102
- tp_recompute_allgather: true
103
- profiler: null
104
- s3_upload: null
105
- tokenizer:
106
- tokenizer_max_length: null
107
- tokenizer_name_or_path: /fsx/marcmarone/upcycle/tokenizer/llama2
108
- tokenizer_revision: null
109
- tokens:
110
- batch_accumulation_per_replica: 8
111
- limit_test_batches: 0
112
- limit_val_batches: 0
113
- micro_batch_size: 4
114
- sequence_length: 4096
115
- train_steps: 6353
116
- val_check_interval: -1