Hugging Face's logo

MIRIBerkeley
/
data-vs-algorithms-ceg-expB

Model card Files
xet
 Community
baileymachihirota commited on
Commit
9fe14ff
·
verified ·
1 Parent(s): 6f9cb66

data-ladder: dl_pythia160m_c4

Browse files
Files changed (1) hide show
  1. data-ladder-runs/dl_pythia160m_c4/run_config.json +120 -0
data-ladder-runs/dl_pythia160m_c4/run_config.json ADDED
@@ -0,0 +1,120 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "config_json": "configs/hf/pythia_160m.json",
3
+ "arm": "dl_pythia160m_c4",
4
+ "data_dir": "/workspace/datasets/c4_neox",
5
+ "neutral_eval_dir": "/workspace/datasets/wiki_eval_union",
6
+ "hf_tokenizer": "EleutherAI/gpt-neox-20b",
7
+ "token_budget": 8870000000,
8
+ "n_epochs": 1,
9
+ "out_dir": "/workspace/runs/dl_pythia160m_c4",
10
+ "global_batch_tokens": 524288,
11
+ "device_batch_size": 32,
12
+ "block_size": 1024,
13
+ "peak_lr": 0.0006,
14
+ "schedule": "cosine",
15
+ "warmup_frac": 0.01,
16
+ "wsd_decay_frac": 0.1,
17
+ "min_lr_ratio": 0.1,
18
+ "weight_decay": 0.01,
19
+ "betas": [
20
+ 0.9,
21
+ 0.95
22
+ ],
23
+ "grad_clip": 1.0,
24
+ "n_checkpoints": 25,
25
+ "first_ckpt_frac": 0.001,
26
+ "warmup_timed_steps": 10,
27
+ "max_steps": 0,
28
+ "eval_device_batch_size": 8,
29
+ "seed": 1234,
30
+ "save_checkpoints": 1,
31
+ "keep_checkpoints": 3,
32
+ "arch": "gptneox",
33
+ "hf_config": {
34
+ "transformers_version": "5.15.1",
35
+ "architectures": null,
36
+ "output_hidden_states": false,
37
+ "return_dict": true,
38
+ "dtype": null,
39
+ "chunk_size_feed_forward": 0,
40
+ "is_encoder_decoder": false,
41
+ "id2label": {
42
+ "0": "LABEL_0",
43
+ "1": "LABEL_1"
44
+ },
45
+ "label2id": {
46
+ "LABEL_0": 0,
47
+ "LABEL_1": 1
48
+ },
49
+ "problem_type": null,
50
+ "vocab_size": 50304,
51
+ "hidden_size": 768,
52
+ "num_hidden_layers": 12,
53
+ "num_attention_heads": 12,
54
+ "intermediate_size": 3072,
55
+ "hidden_act": "gelu",
56
+ "attention_dropout": 0.0,
57
+ "hidden_dropout": 0.0,
58
+ "classifier_dropout": 0.1,
59
+ "max_position_embeddings": 2048,
60
+ "initializer_range": 0.02,
61
+ "layer_norm_eps": 1e-05,
62
+ "use_cache": true,
63
+ "bos_token_id": 0,
64
+ "eos_token_id": 2,
65
+ "pad_token_id": null,
66
+ "tie_word_embeddings": false,
67
+ "use_parallel_residual": true,
68
+ "rope_parameters": {
69
+ "rope_theta": 10000,
70
+ "partial_rotary_factor": 0.25,
71
+ "rope_type": "default"
72
+ },
73
+ "attention_bias": true,
74
+ "is_decoder": false,
75
+ "_name_or_path": "",
76
+ "model_type": "gpt_neox",
77
+ "output_attentions": false
78
+ },
79
+ "world_size": 8,
80
+ "total_steps": 16919,
81
+ "tokens_per_step": 524288,
82
+ "grad_accum": 2,
83
+ "n_params": 162322944,
84
+ "device": "cuda:0",
85
+ "gpu_name": "NVIDIA H100 80GB HBM3",
86
+ "torch_version": "2.10.0+cu128",
87
+ "algorithm": "hf_gptneox",
88
+ "ckpt_steps": [
89
+ 17,
90
+ 23,
91
+ 30,
92
+ 40,
93
+ 54,
94
+ 71,
95
+ 95,
96
+ 127,
97
+ 169,
98
+ 226,
99
+ 301,
100
+ 401,
101
+ 535,
102
+ 713,
103
+ 951,
104
+ 1269,
105
+ 1692,
106
+ 2256,
107
+ 3009,
108
+ 4012,
109
+ 5350,
110
+ 7135,
111
+ 9514,
112
+ 12687,
113
+ 15227,
114
+ 15565,
115
+ 15904,
116
+ 16242,
117
+ 16581,
118
+ 16919
119
+ ]
120
+ }