Hugging Face's logo

iskhare
/
neurips-ablations

Model card Files
xet
 Community
iskhare commited on
Commit
37760e4
·
verified ·
1 Parent(s): 58b8ada

Upload ablation-ncp-sq-corruption-p10 ncp config

Browse files
Files changed (1) hide show
  1. squared16/corruption-p10-50k/ncp/training-config.yaml +196 -0
squared16/corruption-p10-50k/ncp/training-config.yaml ADDED
@@ -0,0 +1,196 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ task: next-concept
2
+ experiment: ablation-ncp-sq-corruption-p10
3
+ experiment_dir: /workspace/ncm/runs/ablation-ncp-sq-corruption-p10/ncp
4
+ model:
5
+ n_layer: 16
6
+ n_head: 16
7
+ bias: true
8
+ dropout: 0.1
9
+ n_embd: 1024
10
+ context_length: 1024
11
+ vocab_size:
12
+ - 16384
13
+ - 16384
14
+ - 16384
15
+ - 16384
16
+ - 16384
17
+ - 16384
18
+ - 16384
19
+ - 16384
20
+ - 16384
21
+ - 16384
22
+ - 16384
23
+ - 16384
24
+ - 16384
25
+ - 16384
26
+ - 16384
27
+ - 16384
28
+ attn_pattern: block_diagonal
29
+ use_positional_encoding: true
30
+ use_level_encoding: true
31
+ prefix_len: 0
32
+ use_rope: true
33
+ rope_base: 100000
34
+ use_qk_norm: true
35
+ post_upsample_conv:
36
+ enabled: false
37
+ kernel_size: 3
38
+ use_relu2: true
39
+ use_flex_attention: true
40
+ shared_output_head: true
41
+ shared_head_mlp: false
42
+ shared_head_mlp_mult: 2.0
43
+ shared_head_mlp_residual: true
44
+ shared_head_mlp_blocks: 2
45
+ n_kv_heads: 4
46
+ mlp_ratio: 3.5
47
+ in_vocab_size: 50304
48
+ predict_level0: true
49
+ bos_token_id: 50256
50
+ tokenizer:
51
+ pre_tokenizer:
52
+ name: hf
53
+ tokenizer: hf
54
+ model_id: gpt2
55
+ special_tokens:
56
+ eos_token: <|endoftext|>
57
+ vqvae:
58
+ n_layers: 6
59
+ vqvae_vocab_size:
60
+ - 16384
61
+ - 16384
62
+ - 16384
63
+ - 16384
64
+ - 16384
65
+ - 16384
66
+ - 16384
67
+ - 16384
68
+ - 16384
69
+ - 16384
70
+ - 16384
71
+ - 16384
72
+ - 16384
73
+ - 16384
74
+ - 16384
75
+ - 16384
76
+ embed_dim: 384
77
+ compression_factor: 1
78
+ in_vocab_size: 50304
79
+ context_length: 1024
80
+ # Exact pinned codec. Never a directory: the sweep must be able to state
81
+ # which VQ-VAE every NCP result was measured through.
82
+ checkpoint_path: ncm/runs/ablation-vqvae-squared16/vqvae/checkpoint-iter-50000.pt
83
+ vector_quantizer_config:
84
+ clss: multiscale_residual_vector_quantizer
85
+ decay: 0.99
86
+ epsilon: 1.0e-05
87
+ commitment_cost: 0.25
88
+ learned_l1_sampling: true
89
+ learned_l1_cascade: true
90
+ learned_l1_cascade_base: 4
91
+ learned_all_sampling: false
92
+ quant_resi:
93
+ enabled: true
94
+ ratio: 0.5
95
+ share_mode: 0
96
+ learnable_ratio: false
97
+ levels:
98
+ use_manual_levels: true
99
+ manual_levels:
100
+ - 1
101
+ - 4
102
+ - 9
103
+ - 16
104
+ - 25
105
+ - 49
106
+ - 81
107
+ - 121
108
+ - 169
109
+ - 225
110
+ - 289
111
+ - 361
112
+ - 484
113
+ - 676
114
+ - 841
115
+ - 1024
116
+ aux:
117
+ fine_drop:
118
+ prob: 0.5
119
+ min_keep: 1
120
+ dropout: 0.2
121
+ pre_quant_groupnorm: 4
122
+ pre_quant_dropout: null
123
+ dataset:
124
+ train_source: data/full_owt/train_gpt2.bin
125
+ validate_source: data/full_owt/valid_gpt2.bin
126
+ wandb:
127
+ project: ncp-colm
128
+ group: ablation-ncp-sq-corruption-p10
129
+ enable: false
130
+ resume: null
131
+ gradients_and_params:
132
+ enable: false
133
+ log: all
134
+ log_freq: 1000
135
+ optimization:
136
+ level_loss_alpha: 0.25
137
+ lr: 0.0005
138
+ reset_lr: false
139
+ warmup_iters: 300
140
+ lr_decay_iters: null
141
+ min_lr: 1.0e-05
142
+ beta_1: 0.9
143
+ beta_2: 0.99
144
+ weight_decay: 0.01
145
+ max_grad_norm: 1.0
146
+ grad_accumulation_steps: 8
147
+ corruption:
148
+ mode: per_level_full
149
+ # Constant corruption rate p = 0.10 at every eligible
150
+ # conditioning level L0..L14. Flat by design: this study sweeps
151
+ # corruption STRENGTH with the shape held constant, so strength is
152
+ # not confounded with shape.
153
+ #
154
+ # L0 is included because under BOS->L0 the model predicts L0 itself
155
+ # and can get it wrong, making it a genuine conditioning level.
156
+ # L15 is target-only (idx_to_ncm_input reads only L0..L14), so its
157
+ # probability must be exactly 0.
158
+ per_level_probs:
159
+ - 0.1000 # L0
160
+ - 0.1000 # L1
161
+ - 0.1000 # L2
162
+ - 0.1000 # L3
163
+ - 0.1000 # L4
164
+ - 0.1000 # L5
165
+ - 0.1000 # L6
166
+ - 0.1000 # L7
167
+ - 0.1000 # L8
168
+ - 0.1000 # L9
169
+ - 0.1000 # L10
170
+ - 0.1000 # L11
171
+ - 0.1000 # L12
172
+ - 0.1000 # L13
173
+ - 0.1000 # L14
174
+ - 0.0000 # L15 target-only
175
+ training:
176
+ seed: 55
177
+ log_interval: 10
178
+ eval_interval: 1000
179
+ checkpoint_interval: 1000
180
+ val_iters: 100
181
+ total_iters: null
182
+ tqdm_interval: 10
183
+ n_epochs: 3
184
+ batch_size: 8
185
+ eval_batch_size: 16
186
+ demo_batch_size: 4
187
+ stop_after_iters: 5000
188
+ keep_last: 100
189
+ torch_compile:
190
+ enable: true
191
+ mode: default
192
+ dynamic: false
193
+ fullgraph: false
194
+ backend: inductor
195
+ mixed_precision:
196
+ enable: true