Back up iclr-debug-vanilla-joint-135b-v2 vanilla-joint step 1272
Browse files- iclr-debug-vanilla-joint-135b-v2/manifest.json +511 -0
- iclr-debug-vanilla-joint-135b-v2/vanilla-joint/config.yaml +219 -0
- iclr-debug-vanilla-joint-135b-v2/vanilla-joint/exports/step-1272/ncp.pt +3 -0
- iclr-debug-vanilla-joint-135b-v2/vanilla-joint/exports/step-1272/vqvae.pt +3 -0
- iclr-debug-vanilla-joint-135b-v2/vanilla-joint/milestone-iter-1272.pt +3 -0
iclr-debug-vanilla-joint-135b-v2/manifest.json
ADDED
|
@@ -0,0 +1,511 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"format": "iclr-debug-production-v1",
|
| 3 |
+
"variant": "vanilla-joint",
|
| 4 |
+
"configs": {
|
| 5 |
+
"vanilla-joint": {
|
| 6 |
+
"task": "joint-next-concept",
|
| 7 |
+
"experiment": "iclr-debug-vanilla-joint-135b-v2-vanilla-joint",
|
| 8 |
+
"experiment_dir": "/home/ishank/mstok-results/iclr-debug-vanilla-joint-135b-v2/vanilla-joint",
|
| 9 |
+
"dataset": {
|
| 10 |
+
"train_source": "/home/ishank/data/full_owt/train_gpt2.bin",
|
| 11 |
+
"validate_source": "/home/ishank/data/full_owt/valid_gpt2.bin",
|
| 12 |
+
"pad_token_id": 50257
|
| 13 |
+
},
|
| 14 |
+
"pre_tokenizer": {
|
| 15 |
+
"name": "hf",
|
| 16 |
+
"tokenizer": "hf",
|
| 17 |
+
"model_id": "gpt2",
|
| 18 |
+
"special_tokens": {
|
| 19 |
+
"eos_token": "<|endoftext|>",
|
| 20 |
+
"pad_token": "<|pad|>"
|
| 21 |
+
}
|
| 22 |
+
},
|
| 23 |
+
"codec": {
|
| 24 |
+
"n_layers": 6,
|
| 25 |
+
"context_length": 256,
|
| 26 |
+
"embed_dim": 384,
|
| 27 |
+
"in_vocab_size": 50304,
|
| 28 |
+
"vqvae_vocab_size": [
|
| 29 |
+
16384,
|
| 30 |
+
16384,
|
| 31 |
+
16384,
|
| 32 |
+
16384,
|
| 33 |
+
16384,
|
| 34 |
+
16384,
|
| 35 |
+
16384,
|
| 36 |
+
16384,
|
| 37 |
+
16384,
|
| 38 |
+
16384,
|
| 39 |
+
16384,
|
| 40 |
+
16384,
|
| 41 |
+
16384,
|
| 42 |
+
16384,
|
| 43 |
+
16384,
|
| 44 |
+
16384
|
| 45 |
+
],
|
| 46 |
+
"compression_factor": 1,
|
| 47 |
+
"dropout": 0.1,
|
| 48 |
+
"pre_quant_groupnorm": 4,
|
| 49 |
+
"pre_quant_dropout": 0.2,
|
| 50 |
+
"vector_quantizer_config": {
|
| 51 |
+
"clss": "multiscale_residual_vector_quantizer",
|
| 52 |
+
"decay": 0.99,
|
| 53 |
+
"epsilon": 1e-05,
|
| 54 |
+
"commitment_cost": 0.25,
|
| 55 |
+
"learned_l1_sampling": true,
|
| 56 |
+
"learned_all_sampling": false,
|
| 57 |
+
"quant_resi": {
|
| 58 |
+
"enabled": true,
|
| 59 |
+
"ratio": 0.5,
|
| 60 |
+
"share_mode": 0,
|
| 61 |
+
"learnable_ratio": false
|
| 62 |
+
},
|
| 63 |
+
"levels": {
|
| 64 |
+
"use_manual_levels": true,
|
| 65 |
+
"manual_levels": [
|
| 66 |
+
1,
|
| 67 |
+
4,
|
| 68 |
+
9,
|
| 69 |
+
16,
|
| 70 |
+
25,
|
| 71 |
+
36,
|
| 72 |
+
49,
|
| 73 |
+
64,
|
| 74 |
+
81,
|
| 75 |
+
100,
|
| 76 |
+
121,
|
| 77 |
+
144,
|
| 78 |
+
169,
|
| 79 |
+
196,
|
| 80 |
+
225,
|
| 81 |
+
256
|
| 82 |
+
]
|
| 83 |
+
},
|
| 84 |
+
"aux": {
|
| 85 |
+
"fine_drop": {
|
| 86 |
+
"prob": 0.5,
|
| 87 |
+
"min_keep": 1
|
| 88 |
+
}
|
| 89 |
+
}
|
| 90 |
+
}
|
| 91 |
+
},
|
| 92 |
+
"generator": {
|
| 93 |
+
"n_layer": 12,
|
| 94 |
+
"n_head": 12,
|
| 95 |
+
"bias": true,
|
| 96 |
+
"dropout": 0.1,
|
| 97 |
+
"n_embd": 768,
|
| 98 |
+
"context_length": 256,
|
| 99 |
+
"vocab_size": [
|
| 100 |
+
16384,
|
| 101 |
+
16384,
|
| 102 |
+
16384,
|
| 103 |
+
16384,
|
| 104 |
+
16384,
|
| 105 |
+
16384,
|
| 106 |
+
16384,
|
| 107 |
+
16384,
|
| 108 |
+
16384,
|
| 109 |
+
16384,
|
| 110 |
+
16384,
|
| 111 |
+
16384,
|
| 112 |
+
16384,
|
| 113 |
+
16384,
|
| 114 |
+
16384
|
| 115 |
+
],
|
| 116 |
+
"attn_pattern": "block_diagonal",
|
| 117 |
+
"use_positional_encoding": true,
|
| 118 |
+
"use_level_encoding": true,
|
| 119 |
+
"prefix_len": 0,
|
| 120 |
+
"use_rope": true,
|
| 121 |
+
"rope_base": 10000,
|
| 122 |
+
"use_qk_norm": true,
|
| 123 |
+
"post_upsample_conv": {
|
| 124 |
+
"enabled": true,
|
| 125 |
+
"kernel_size": 3
|
| 126 |
+
},
|
| 127 |
+
"use_relu2": true,
|
| 128 |
+
"use_flex_attention": true,
|
| 129 |
+
"shared_output_head": true,
|
| 130 |
+
"shared_head_per_level_bias": false,
|
| 131 |
+
"shared_head_per_level_scale": false,
|
| 132 |
+
"shared_head_adapter_rank": 0,
|
| 133 |
+
"single_shared_projection": true
|
| 134 |
+
},
|
| 135 |
+
"objectives": {
|
| 136 |
+
"codec_weight": 1.0,
|
| 137 |
+
"ncp_weight": 1.0,
|
| 138 |
+
"soft_assignment_temperature": 1.0
|
| 139 |
+
},
|
| 140 |
+
"optimization": {
|
| 141 |
+
"codec_lr": 0.001,
|
| 142 |
+
"codec_min_lr": 0.0001,
|
| 143 |
+
"codec_warmup_iters": 0,
|
| 144 |
+
"codec_lr_decay_iters": 171662,
|
| 145 |
+
"generator_lr": 0.0005,
|
| 146 |
+
"generator_min_lr": 1e-05,
|
| 147 |
+
"generator_warmup_iters": 300,
|
| 148 |
+
"generator_lr_decay_iters": 171662,
|
| 149 |
+
"beta_1": 0.9,
|
| 150 |
+
"codec_beta_2": 0.99,
|
| 151 |
+
"generator_beta_2": 0.99,
|
| 152 |
+
"weight_decay": 0.1,
|
| 153 |
+
"max_grad_norm": 1.0,
|
| 154 |
+
"level_loss_alpha": 1.0,
|
| 155 |
+
"grad_accumulation_steps": 6,
|
| 156 |
+
"corruption": {
|
| 157 |
+
"mode": "per_level",
|
| 158 |
+
"per_level_probs": [
|
| 159 |
+
0.85,
|
| 160 |
+
0.8321428571,
|
| 161 |
+
0.8142857143,
|
| 162 |
+
0.7964285714,
|
| 163 |
+
0.7785714286,
|
| 164 |
+
0.7607142857,
|
| 165 |
+
0.7428571429,
|
| 166 |
+
0.725,
|
| 167 |
+
0.7071428571,
|
| 168 |
+
0.6892857143,
|
| 169 |
+
0.6714285714,
|
| 170 |
+
0.6535714286,
|
| 171 |
+
0.6357142857,
|
| 172 |
+
0.6178571429,
|
| 173 |
+
0.6
|
| 174 |
+
],
|
| 175 |
+
"skip_level0": true
|
| 176 |
+
}
|
| 177 |
+
},
|
| 178 |
+
"training": {
|
| 179 |
+
"log_interval": 10,
|
| 180 |
+
"eval_interval": 1000,
|
| 181 |
+
"checkpoint_interval": 1000,
|
| 182 |
+
"eval_batch_size": 4,
|
| 183 |
+
"val_iters": 25,
|
| 184 |
+
"keep_last": 3,
|
| 185 |
+
"seed": 55,
|
| 186 |
+
"codec_initialization_seed": 42,
|
| 187 |
+
"generator_initialization_seed": 55,
|
| 188 |
+
"total_iters": 171662,
|
| 189 |
+
"batch_size": 64,
|
| 190 |
+
"expected_world_size": 8,
|
| 191 |
+
"expected_global_batch_size": 3072,
|
| 192 |
+
"resume_checkpoint": null,
|
| 193 |
+
"milestone_steps": [
|
| 194 |
+
1272,
|
| 195 |
+
6358,
|
| 196 |
+
8000,
|
| 197 |
+
17167,
|
| 198 |
+
34333,
|
| 199 |
+
68665,
|
| 200 |
+
102997,
|
| 201 |
+
137330,
|
| 202 |
+
171662
|
| 203 |
+
],
|
| 204 |
+
"generation_eval_steps": [
|
| 205 |
+
8000,
|
| 206 |
+
17167,
|
| 207 |
+
34333,
|
| 208 |
+
68665,
|
| 209 |
+
102997,
|
| 210 |
+
137330,
|
| 211 |
+
171662
|
| 212 |
+
]
|
| 213 |
+
},
|
| 214 |
+
"torch_compile": {
|
| 215 |
+
"enable": true,
|
| 216 |
+
"scope": "loss",
|
| 217 |
+
"mode": "default",
|
| 218 |
+
"dynamic": false,
|
| 219 |
+
"fullgraph": true,
|
| 220 |
+
"backend": "inductor"
|
| 221 |
+
},
|
| 222 |
+
"mixed_precision": {
|
| 223 |
+
"enable": true
|
| 224 |
+
},
|
| 225 |
+
"wandb": {
|
| 226 |
+
"entity": "mstok",
|
| 227 |
+
"project": "iclr-debug",
|
| 228 |
+
"group": "full-owt-ctx256-16sq-135b-v1",
|
| 229 |
+
"enable": true,
|
| 230 |
+
"id": "iclr-debug-vanilla-joint-135b-v2-vanilla-joint",
|
| 231 |
+
"resume": "allow",
|
| 232 |
+
"gradients_and_params": {
|
| 233 |
+
"enable": false,
|
| 234 |
+
"log": "all",
|
| 235 |
+
"log_freq": 1000
|
| 236 |
+
}
|
| 237 |
+
},
|
| 238 |
+
"export": {
|
| 239 |
+
"vqvae_config_template": "/home/ubuntu/mstok/config/repro-ctx256/vqvae.yaml",
|
| 240 |
+
"ncp_config_template": "/home/ubuntu/mstok/config/repro-ctx256/ncp-sharedhead.yaml"
|
| 241 |
+
},
|
| 242 |
+
"iclr_debug": {
|
| 243 |
+
"stage": "vanilla-joint",
|
| 244 |
+
"target_positions": 135000000000,
|
| 245 |
+
"lr_horizon_positions": 135000000000,
|
| 246 |
+
"stop_step": 171662,
|
| 247 |
+
"pilot": false,
|
| 248 |
+
"tokenizer_checkpoint": null,
|
| 249 |
+
"tokenizer_steps": null,
|
| 250 |
+
"budget_unit": "input positions including padding; log non-padding tokens separately"
|
| 251 |
+
}
|
| 252 |
+
}
|
| 253 |
+
},
|
| 254 |
+
"sources": {
|
| 255 |
+
"config/repro-ctx256/alignment-generator.yaml": "f62f36d078a3a373a19ea716a3ebc7eda5ae2fb7d670c5b935c734826bb76895",
|
| 256 |
+
"config/repro-ctx256/alignment-joint.yaml": "bb511950d20335e4237e406464ab26760a7fd9c9c9472c0507addd88d33d5178",
|
| 257 |
+
"config/repro-ctx256/alignment-tokenizer.yaml": "2d32955296f7d15990f284328c1cf8f52071abdba28033a65e272bb49cf5dd56",
|
| 258 |
+
"config/repro-ctx256/alignment.yaml": "3a486fa2ed28eca4fc406232669399b0625d274376ac43271f4ccf187b1e9077",
|
| 259 |
+
"config/repro-ctx256/joint.yaml": "8b4889111687a52e2d7d71995bda45a7f01f2777674c88191f338dca4c1732d2",
|
| 260 |
+
"config/repro-ctx256/mstok-semantic-eostok-matched10ep.yaml": "ae8a72c0c9f1cb5d2e301d83a1cdc08985a1f895bf627a5f75b1b3ce627f5e58",
|
| 261 |
+
"config/repro-ctx256/mstok-semantic-eostok.yaml": "2c9c48b21325c0e479990701b28116d78f611101724846747d173e52145fe15a",
|
| 262 |
+
"config/repro-ctx256/mstok-semantic-gear-matched10ep.yaml": "164cb8ac39fa407bb337f559ad8a9c62a34708fb900b8829d3ccf182961f2d0c",
|
| 263 |
+
"config/repro-ctx256/mstok-semantic-gear.yaml": "8cb8c998551daef0bf7fd9ec77b13b34c06662028d8820767e1432b5a9e8be54",
|
| 264 |
+
"config/repro-ctx256/mstok-semantic.yaml": "e0b00dbfa3eb838103f657c19bde20e36375c7fbe692623452ea093003f1ac98",
|
| 265 |
+
"config/repro-ctx256/mstok-w1-pilot.yaml": "21ee22eeccaaea2563c92ccc84f6df3871ea5da6a5ce980656088ea577dc1438",
|
| 266 |
+
"config/repro-ctx256/mstok-w1.yaml": "b6bf6079114b3c997fab89e85a53318bb2ec5dae55678513599d84152eab701b",
|
| 267 |
+
"config/repro-ctx256/mstok.yaml": "d9bb216f4eeb58857d72be399d5f8847d197ce70d0e504879f0788045483dc87",
|
| 268 |
+
"config/repro-ctx256/ncp-sharedhead.yaml": "86c9efbcb2b4f74b7253aa3bc4393de4a6a05e20ea30da573d1ca094ec600645",
|
| 269 |
+
"config/repro-ctx256/ncp.yaml": "93663654c4ce5ad0931db052edf507fdcb7c0ff9fc9722f4018820deee5c5acf",
|
| 270 |
+
"config/repro-ctx256/substitution-generator.yaml": "064f55887b9b2b8f3b0b52c094d4a193510dc1148ad7a5c7f53e317971f87c7c",
|
| 271 |
+
"config/repro-ctx256/substitution-joint.yaml": "b1576f4ecfe6fc6029a0cd868271b80a1d988b7f3667c6b1a8d24a6a4015df3b",
|
| 272 |
+
"config/repro-ctx256/substitution-tokenizer.yaml": "e29be5046f0fafbfe4c06973be4cd036a004e845348911cc0e6e05a7fca5d425",
|
| 273 |
+
"config/repro-ctx256/substitution.yaml": "ebba1c7e664de0e84913bdfd38e794837781c42c1b48cde90135ff9417856a95",
|
| 274 |
+
"config/repro-ctx256/vqvae.yaml": "05fde46e57c3e6b411e8a6afcac251f178e4b8a0856a0b99332bb284fb4710ce",
|
| 275 |
+
"data/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
|
| 276 |
+
"data/serialized_dataset.py": "5622c2d94236cd1b9b4ca18e8d5c272a86b54d193d99048fb9b7cd2687e8dbc6",
|
| 277 |
+
"data/tinysentences/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
|
| 278 |
+
"data/tinysentences/clean.py": "918a9a91aae484a5e06d6d212c2c0974946a4032e061c62a7b65851735bfc8dd",
|
| 279 |
+
"data/tinysentences/clean_parsed.py": "009a087f967e171bc6098484dbfd58635302900f6b07629629898f1d00fa70a3",
|
| 280 |
+
"data/tinysentences/generate_meta.py": "662714f1370e05567d412add168a7c3a05f4b1c9adb047fad374900bc1936a7e",
|
| 281 |
+
"data/tinysentences/generate_metadata_char.py": "afd3336ea03ff0653ccaf41c121d9b93debf7a380db6cd881b62b308ee6ee68c",
|
| 282 |
+
"data/tinysentences/generate_training_data_word.py": "82e2edcb7ea4db0609ccf2b047a0b207e7b8fcff4fc0b4ceee521ff278e8a8e8",
|
| 283 |
+
"data/tinysentences/parse_summaries.py": "2317159364dbbe7206f252d2939cab5d375395e0a536be0a1f60c5df88b31702",
|
| 284 |
+
"data/tinysentences/prepare.py": "35263b287f93337fda4207ed1991e9e8a820ab75139411bb7dd5f0a8a25bcbd9",
|
| 285 |
+
"data/tinysentences/sentences_only.py": "5e4d72bf612df27dad46cf606a40402606d897043ae24274dc2dfa88ec0b3e79",
|
| 286 |
+
"data/tinysentences/stats.py": "2665e9e6fac4a8f6c3e5bd8b07cbfd941ba6359a43981d89bdef58c136fdd3a9",
|
| 287 |
+
"data/tinysentences/summarize/merge_summaries_char.py": "bce94f4d9a824c2d25e45448023d59b7fbc42aba22b33efbc067cdbba93e1934",
|
| 288 |
+
"data/tinysentences/summarize/merge_summaries_word.py": "f6079110332cc38b2dd3571c30bedcca97098c39498f45eec1ef6e60cce7ac81",
|
| 289 |
+
"data/tinysentences/summarize/summarize_gemini.py": "282a438efce0dee351fd9833a59c56e4480bed4314dae40844201794ada3bd96",
|
| 290 |
+
"data/tinysentences/summarize/summarize_gpt.py": "fc08cbc37d6161e5b0057a1ede43272d96f7274524917d6c4d42c22f505c6e09",
|
| 291 |
+
"data/tinysentences/summarize/summarize_llama.py": "7f3f8230652a5b88ea2856b273d497cd5b633580b430c909e4f996cd0232832d",
|
| 292 |
+
"data/tinysentences/summarize/summarize_qwen.py": "b07e3c226ab0fdf5b21b025263adfa7c2625d640f0fac3c787bc4a2c9e32107f",
|
| 293 |
+
"data/tinysentences/summarize/summarize_together.py": "8c26ea052f2a64c2653d39f1ea5c13983dfbc9547f21cd7d7ea9d9b39033bffe",
|
| 294 |
+
"data/tinysentences/test.py": "9749244da77c078f550a78751664b45bb92067682f3aa27c2338419f1186c844",
|
| 295 |
+
"data/tinysentences/test_regex.py": "46652e189a0d5f5c0c7a2c90b857082004bf6727ef1d3ad42b631ac7bb0aa88f",
|
| 296 |
+
"data/tinysentences/tinysentences_char_dataset.py": "152ab9c30aec669a9db14df9e90412597870105eff9cfb737b876aad2323a206",
|
| 297 |
+
"data/tinysentences/tinysentences_dataset.py": "f90251d21871f6a7ad5ebd2ad0a6e59cd0b02ca245f7e46f93e6d970bf657bc7",
|
| 298 |
+
"data/tinysentences/tinysentences_vae.py": "30f7adde5413348faf5066448706747fe26a7fedd510ffea720aa0bf8283f6f8",
|
| 299 |
+
"data/tinysentences/to_csv.py": "c06c74648ed95b4494835c631c94fe1a00a9ff47a272f5e38546a7f94ff9be8b",
|
| 300 |
+
"data/tinysentences/tokenize/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
|
| 301 |
+
"data/tinysentences/tokenize/common.py": "e877c74df2a48d15991492b126cd0c5fe26731d61f3aa36d83ebf2cb9bda34a0",
|
| 302 |
+
"data/tinysentences/tokenize/tokenizer.py": "c16abc50b92a163f6978542b36b03c9a14aa00153c73117c4d52084d5de25c2a",
|
| 303 |
+
"data/tinysentences/tokenize/train_bpe.py": "fa8e55978dd5ca90117200648b468fb22822bf6981f15f0394c815e72eae1bdb",
|
| 304 |
+
"data/tinystories/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
|
| 305 |
+
"data/tinystories/download.py": "5afce37e912f6965b6722f0bc028076a655fea78304f1a213e4862dbe223cdcb",
|
| 306 |
+
"data/tinystories/stats.py": "216650fe19b78e8431c4512ff87a5c84af64d9e56ab8fdaf3952ca035725ea1b",
|
| 307 |
+
"data/wikitext/parse_wikitext.py": "ab583e0328f82afc3ba242410e8076be42ef1a16a51d66f3eb46a713cca591d0",
|
| 308 |
+
"evaluation/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
|
| 309 |
+
"evaluation/eval_random_gen_ppl_checkpoint.py": "ab5ffe43a59e8ad6942994250986071327894441312deb8a16e9241dc5d51df9",
|
| 310 |
+
"evaluation/gen_ppl.py": "e6853b14d27f93407505eb7607a1bd9fed49a7add1511ceca2b4a3076d533859",
|
| 311 |
+
"evaluation/gen_ppl_ncp.py": "96d68adee372e5186d0470659171eb03829e3d37f611e8f5917b45fb61e3a4ad",
|
| 312 |
+
"evaluation/gen_ppl_ntp.py": "b2638622bc041fa88e9d538be4229b3723ebec19c5b99fd6e5b73db4776dc952",
|
| 313 |
+
"evaluation/guess.py": "c27803225dd21b2deff1e7e494b9e3666180b93776340164ce6e953dacc50656",
|
| 314 |
+
"evaluation/ppl.py": "f3b581cb4849b0b2aa518729b8338838c7055697c3a26028dedb819c335ac838",
|
| 315 |
+
"evaluation/run_genppl_multiseed.py": "f8c9ac982f4caa4335b5f4d1a22d70f56dc11a706106a3dfb31d344eb8c93c2e",
|
| 316 |
+
"evaluation/run_genppl_release_ts.py": "d1bc5c00089118d9cf6c05c77459329ad40c2f8bc347b5aeedda8f5318de8796",
|
| 317 |
+
"evaluation/run_genppl_sweep.py": "ade4f39e373125848fdd32938e8d7bf86a0ec5108cca9bd75f0a966d092a798e",
|
| 318 |
+
"evaluation/show_8k_samples.py": "30ad825d5a24375e178750dfaaac45bc486bc3587f2e7d6d305d53bf37f8d226",
|
| 319 |
+
"evaluation/show_low_sample.py": "d696f1e54ca67e87565dce5864e0e6e2ad4194108cfb1c9c1e06eebd97008da6",
|
| 320 |
+
"evaluation/summarize_gen_ppl.py": "307f3d0d85884879295d5268aaf7f399f654b1b330aaa5728d6824ad24c6d15a",
|
| 321 |
+
"evaluation/sweep_gen_ppl_checkpoints.py": "3a9141e27d84b49c5493194dfc5a70bfa0a915cdf5ceb7a46afcf27cc6f33acb",
|
| 322 |
+
"evaluation/sweep_gen_ppl_ncp.py": "d91c2959cfed51d224ccb1c0d0f2c3b9a35c4027cc9ab8c65aa7603d587de761",
|
| 323 |
+
"evaluation/sweep_gen_ppl_ncp_scaling.py": "05249c1e00c5eb05042b6ea5b1ca008ecb8f0e4a77537ee755a27c153c25a17c",
|
| 324 |
+
"evaluation/sweep_gen_ppl_ncp_yolo.py": "4b26ef25b27af2f7fcbcb5ff3c312d47d2a4746ba759155d28802e978a681755",
|
| 325 |
+
"evaluation/sweep_gen_ppl_ntp.py": "14a04074741ee44580ec6557870d42534dd172d26c5be3e943ed5baf1b727303",
|
| 326 |
+
"evaluation/sweep_random_llama_yolo.py": "4062b54289fcd9407d15feb1ffd241b8722fd717c700350c22472fecec5ae46c",
|
| 327 |
+
"evaluation/val_oracle_ppl.py": "1a23f3c2897b6a6a3d239ed533979de35352d303bae094f4006bef4156f935c0",
|
| 328 |
+
"evaluation/val_vqvae_ppl.py": "4c293c502d3faf982d4ad66566b7a037faa7d28191d929de1621cb67f5ae8b5f",
|
| 329 |
+
"models/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
|
| 330 |
+
"models/common.py": "99aa0da356963a282936413d738ee1da72a3e3f277fecc28eddb553051582a4b",
|
| 331 |
+
"models/masked_lm.py": "80279614d74efc7b4c25306789d74e2f71045c54e4e4f485ee365f664705d7f6",
|
| 332 |
+
"models/next_concept.py": "c8c4abd6440bb6a35e42fdda8d3d2cf5ad506e74ce3494858c37bd2561526052",
|
| 333 |
+
"models/next_token.py": "d0d292b9f5c032a351cbca1a8f0a6d41c3062890cb3e3f5b67d4e23a7cffeb77",
|
| 334 |
+
"models/quant.py": "b7b73abd4e55e10454cb680dea82124c38c222d37236c4c76b5c7d21123ac1b1",
|
| 335 |
+
"models/semantic_input.py": "b06658d4bc641397cbb973255eadfd2d6087fde244d801d3c70d29e3ce841bf2",
|
| 336 |
+
"models/vae.py": "02e2213a8878920374baf27dd10dfc03847dce866557c97e816c427d230c9ded",
|
| 337 |
+
"models/vqvae.py": "8eb3503eb1d990121b7d6138a4f8216aecfc72246d4a92bbecf98025c8a3d504",
|
| 338 |
+
"scripts/benchmark_iclr_debug.py": "87931ab0bffee37f1973a8a04952b843eb398157c6b6f8fb8a477d877d84b443",
|
| 339 |
+
"scripts/benchmark_mstok_compile.py": "aad909da6f05ba25ac3d8f6b3cf450103bed55d0b93b1490764c0ef54f35b606",
|
| 340 |
+
"scripts/eval_owt_ckpts_gpt2_large.py": "4977a2d614698f60032fb6fd7fef614eb503d5b3883f499146a4fc048214a509",
|
| 341 |
+
"scripts/evaluate_mstok_sidecar.py": "9ba33e865fa0fad34bd2ab375d7fe38f37c441e2d96348f0e389e29dd9643455",
|
| 342 |
+
"scripts/evaluations/sweep_gen_ppl_shared_vocab_ncp.py": "2311ac71f9beba282426faa21390a5898c7d183a95f529755d6e1d6e8fc24050",
|
| 343 |
+
"scripts/neurips/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
|
| 344 |
+
"scripts/neurips/eval_gen_ppl_ncp_owt.py": "b92eff1ce198eb4ac8c88da7113b41d095c12acad92db541685175993db3535c",
|
| 345 |
+
"scripts/neurips/sweep_gen_ppl_owt_gpt2.py": "02adb1629cfa40ff51a75152e4da9c9ee00ead42b7ecda9e78cf7101e6826a67",
|
| 346 |
+
"scripts/preflight_owtsmall_mstok_ctx256.py": "a4f80921435b6cd136d089ec39bfa5442359d0dffbeefaaa1019178b8364657a",
|
| 347 |
+
"scripts/prepare_full_owt.py": "b1cb6b74397625c2b0643c4ee744ca3c89dfa6bbb7eeb7e9681e3a36f91a2c24",
|
| 348 |
+
"scripts/r1_lab/diagnose_at_40k.py": "b1582dcc29ccebc9255eaa77a7d56a7dbdef589ec5d5a9b0af150dede98874ab",
|
| 349 |
+
"scripts/r1_lab/inspect_summaries.py": "4154e8a6c1b3f68df5c623a57a1a7632b594c46c5f6d1d85cb08c476d6aa9cea",
|
| 350 |
+
"scripts/r1_lab/overnight_pipeline.py": "53f4e03ce1a5fb43fbca3ae455288f45ae8350b0a16f3d5318763011de6a8f96",
|
| 351 |
+
"scripts/r1_lab/prime_rope_qknorm_score_cache.py": "a791cdec03b415bfccdfd8d89c7e659aff8a2a5696d16ab7b4350ff608d0391b",
|
| 352 |
+
"scripts/r1_lab/sweep_gen_ppl_r1lab.py": "c7c0acf9aa6f07bb0a985cf0d5e5692c696e658ebb0470b7c92d9332565f9c06",
|
| 353 |
+
"scripts/r1_lab/sweep_gen_ppl_r1lab_v15.py": "524aa8680aa1b9fea2689509848c19e5a74a3ad72e78b0388bbca7acd43be22e",
|
| 354 |
+
"scripts/r1_lab/sweep_gen_ppl_rope_qknorm.py": "7822df057355b0a3a71ea5145742107c6e30f906859096237ff61d765fe36d50",
|
| 355 |
+
"scripts/r1_lab/train_variant.py": "9d0a2091784609ac9728226afd82f56fd4c1ac2dc08e89e1b17b60d34bb3d55e",
|
| 356 |
+
"scripts/run_alignment.py": "6312cd9d0976899763589fdcbd258c7681f6e9e96d7396cf35ff022d4695b862",
|
| 357 |
+
"scripts/run_iclr_debug.py": "52024e054dc9bf642b8e094b91659ae19ce41cc3c61bd79038b5954970c42806",
|
| 358 |
+
"scripts/run_mstok_semantic.py": "8a7a38a50dd6694950bfa0e7b79718ec653c7e70be99fe13cbf2ee05c28f46a0",
|
| 359 |
+
"scripts/run_mstok_w1_pilot.py": "35224e462d0d11eb3e4326b0daa8bad7c99e1e30cdc6383310f8e2c8b5bb708f",
|
| 360 |
+
"scripts/run_substitution.py": "2cd22eaa58dab8b5231984eee340a3a2bdd0b88cc83333f7133447471e7f73f1",
|
| 361 |
+
"scripts/smoke_test_masked_l0.py": "d24f1270b3219a31212b68bcbb786f649c7c72f96e53f602d87ad6c17f3f73e5",
|
| 362 |
+
"scripts/test0_diagnostics.py": "ed389c666beef015ccac004f314a5a1428def303b46ac0e442f0236337344549",
|
| 363 |
+
"scripts/test0_r1_similarity.py": "4a486878bebd418a3b5454ddbf02585726ccbacc2700fdf591aa524766f9c9c9",
|
| 364 |
+
"scripts/test0_r1_uniqueness.py": "3ee59d4397d5b268151dbb8ca779c7f9bf8461d129d97fcd1a517c48ec2a69a1",
|
| 365 |
+
"scripts/test0_residual_norms.py": "3c4f68e5960e84796dc0bca4fd4583bf3b598552654f9b565fd5fcabd420f9e1",
|
| 366 |
+
"scripts/test_gen_equiv.py": "ae0e9c513422ee58bdc050ed6218ee54fc555dfb15ea8968c9ffc62a7a7ea99a",
|
| 367 |
+
"scripts/training_budget.py": "bb94c19c6dfbdd542946b10476b8d6958b8a189abd141cf62288fdb5fa9a2d38",
|
| 368 |
+
"scripts/vqvae_utilization.py": "5f90327f1507a5042b2f77b119fe4fc0df4e2131e5bbc21069dffd1fadc059c5",
|
| 369 |
+
"scripts/xsum/codec_ceiling.py": "ddb751e9152d5c88966e588d89cdceee4c5d91c4139aaedf369f65a45ecf2276",
|
| 370 |
+
"scripts/xsum/eval_merge.py": "6e313d6ed25227d0418b7422d58e09ceda4fc6dfae7d0fd663417b306241135c",
|
| 371 |
+
"scripts/xsum/eval_q0.py": "82fe04dccce28d7fc9c6ec8b4c9c91404a49527f9437913f44f51a6853ba9abc",
|
| 372 |
+
"scripts/xsum/eval_shard.py": "07192f6ba6219da1e8123ceeb99748c86da85322859ed5fa25cebcc3816f7ea8",
|
| 373 |
+
"scripts/xsum/eval_shard_cfg.py": "95ac4a8e2e3a73a21fa4303c62df80c086f23bcfe5ab34eb07ae90b585052ecb",
|
| 374 |
+
"scripts/xsum/eval_test.py": "7f92d578c31893e788e5fa7d9eef3098ba5602b3642873ce2ce031342c8be868",
|
| 375 |
+
"scripts/xsum/eval_uncond_rep.py": "ba4740b29bc091d866e2dfd91881e9a7d5c46a1bfa993d13eb63c0467397ec28",
|
| 376 |
+
"scripts/xsum/make_warmstart.py": "78793d6ef11fbf658c2837ef12cb843fc4817add8ba73c822b5e408901de3318",
|
| 377 |
+
"scripts/xsum/mbr_fast.py": "3db6e21d2dcce2e3bcf6f22474535d7eba0232b071b4271d2b172f787aed5c85",
|
| 378 |
+
"scripts/xsum/mbr_select.py": "107a061014c62395b7f5fd986fba9e55dd738054e2474022005cb7e9e0c9b8fb",
|
| 379 |
+
"scripts/xsum/mbr_shard_cfg.py": "3e7162375028c391276149eff288e9de40c874a3108ac53de0248198f4d08791",
|
| 380 |
+
"scripts/xsum/pool_append.py": "e50252f48c4d8d905be0de3eb56ca1d720f8f8545c6fdcb58ed27f0360c4e2ad",
|
| 381 |
+
"scripts/xsum/prep_xsum_le48.py": "3688ed1abc49d18f75d34f8382ff68a5afbd753b36cefdc0652d0bffb6a5f347",
|
| 382 |
+
"scripts/xsum/prep_xsum_mix30.py": "4bd47e52637c4b6bca5fd4956c1787b506a7f51b8a5bb0b68f19f728c4d48afb",
|
| 383 |
+
"scripts/xsum/prep_xsum_mix_split.py": "ec2fe74f08068ae8fda4ed59a86537a1d3f48276d205099bbfe16f8428ca36d5",
|
| 384 |
+
"scripts/xsum/prep_xsum_pass_uncond.py": "82ea50248d811e5182e63d0b4dfd6f995dd2f47eb963b5bb1c5d09bc533db7c1",
|
| 385 |
+
"scripts/xsum/prep_xsum_test_all.py": "7389fbdf5612ba524801c5b04109f810b139152ca50101b60acbb9b1e767b256",
|
| 386 |
+
"scripts/xsum/score_checkpoints.py": "41b412c06cd92cbd4caa6b586da7ddc7e913607a47c1a3aac0ee0a36a20a36be",
|
| 387 |
+
"tokenization/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
|
| 388 |
+
"tokenization/bpe_tokenizer.py": "0c8af84868a6aa40f981768f8ab49d28e501d55cce444edb1ad315de53c3445a",
|
| 389 |
+
"tokenization/build_word_vocab.py": "778f9666f4e7c642dc471773ee81c92c26975db37de1ebfffb605cae211351b5",
|
| 390 |
+
"tokenization/byte_tokenizer.py": "eb567a1772ae69738884ed9b94330aa39387c49e54d76e4b9752e67674fad810",
|
| 391 |
+
"tokenization/compression_ratio.py": "ed07ab1cc38d6176362631ce655bc974dd3623b87693f51279d5047e6bc6890f",
|
| 392 |
+
"tokenization/hf_tokenizer.py": "2ca8fae097f9c640bcb52b4788c37de8e0d9bb2fa971e30a5abb39f90ff97d4c",
|
| 393 |
+
"tokenization/tokenize.py": "f8ab54338613b6bd84513fdda85be173fd32274c35d97ef39a5c17d584f8108f",
|
| 394 |
+
"tokenization/tokenize_doc_level.py": "a06c81cf088fb81d2317ed1c3a1417a0e48ae0ea04962824f13f5d3beddc8902",
|
| 395 |
+
"tokenization/tokenize_vqvae.py": "14730eec3da87f6528e5fbd1166d5da79e10ac1700f7cd0be73bc40716bdebfc",
|
| 396 |
+
"tokenization/tokenizer.py": "e527d47e4ed8634c673d48a872a07c9cc8828d30988331de2d4bd3b7c21ddcb7",
|
| 397 |
+
"tokenization/train_bpe.py": "bf4e2a29eeba874024ed95f32fe720ae2320f72cb978361692db73e71be98683",
|
| 398 |
+
"tokenization/vqvae_tokenizer.py": "d7fa9d595865aa3e0b0d185bfd04721eb297fe5f657c4fca4ce69fc96aca308f",
|
| 399 |
+
"tokenization/word_tokenizer.py": "a405621bf35ed6cbb29abadd9d1d651e0783e6801fc7645b35e799d5de381ac2",
|
| 400 |
+
"train_mstok.py": "077b7c99c12daad2bbd986918d0f487411779de05a472af71124098da399b79d",
|
| 401 |
+
"trainer/alignment_trainer.py": "54f8ee7cca4d407223f2ce5146e6fc561ad748e20f2c702beb12ae040cd290c4",
|
| 402 |
+
"trainer/corruption.py": "f0f24c94652e6e9cf8e4ef30ae1a302e3232d64e765c931696d454db96e8b57d",
|
| 403 |
+
"trainer/joint_trainer.py": "c0c25a07338930a3e5e8680ef95f07e9ccb2573587c0db1ec1854963f4cdd8f8",
|
| 404 |
+
"trainer/mstok_trainer.py": "90712e396472d5dac9679b62af4f0b62afa6390a839d4d9bd9dae57406fe76a4",
|
| 405 |
+
"trainer/ncp_trainer.py": "73fc23f78f403dc39800b924dfb18ad42ffeab847fed30e931183c3d3acaee68",
|
| 406 |
+
"trainer/ntp_trainer.py": "5076d7aa29fac70869e1e309a15af6c4d3a91d6de8e00ac113a586841a9866e1",
|
| 407 |
+
"trainer/onpolicy.py": "6ff1fe4103d71001afe2d73e574d20c7242381e420929db14020141464ec6821",
|
| 408 |
+
"trainer/plain_two_stage.py": "4c5973ed87100b1b478aa1df0bb16cd3d1eb5844dffe5894003dd67c463f6c9a",
|
| 409 |
+
"trainer/scheduled_sampling.py": "f53515e23c2cec1231265f21a341ff77d11fa8a6129e0e382d540cc85571f5ed",
|
| 410 |
+
"trainer/semantic_mstok_trainer.py": "ebc02e80d279a445c99d8f09505cdfaf7632f1a661d6880f09cc1bd0fa90ef30",
|
| 411 |
+
"trainer/semantic_teacher.py": "7983b5da86d25954ed8267eee03d144c421305967705e0a242f4f025794ceeae",
|
| 412 |
+
"trainer/substitution_trainer.py": "33c7833b2e8dfabbb9ee56b258708055d3194b8d7453a1b8d93f59b44ff91465",
|
| 413 |
+
"utils/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
|
| 414 |
+
"utils/arg_utils.py": "e9eb7971f6273e9630b56fd87a03725b29183a8c80abb014c4c84f8b1bdb6445",
|
| 415 |
+
"utils/benchmark.py": "ccf436ef72a4e626c7f671d8e63a683057bf502a3852e5a7142410d1dc3e5501",
|
| 416 |
+
"utils/data.py": "78cce11e777813755cd171ccaa3247f89ab99ac4079e0157e55c72ce8a5a7fe5",
|
| 417 |
+
"utils/dist.py": "b2849d3ce03a8b1737e3dd61216e76e4a9e82d63ad1ea74cddf72e2285eaca64",
|
| 418 |
+
"utils/experiment_config.py": "616a70fefaac99ab0d856a1b6c36787a28fb9af8a4c55fc6d4765012fbaaeb38",
|
| 419 |
+
"utils/iclr_training.py": "7f59410a73f3b903e66811e85cfcec225871dd5825e156eba1c75347da746927",
|
| 420 |
+
"utils/logging.py": "901e67515cc65a35f575f05d16a3b9ef4a5b2cb0ea252cd7abbc80ecce9e88e8",
|
| 421 |
+
"utils/lr.py": "4d1ea2c25340afc4bdde8b582dab36f174e0b2965fa7bafa2a7539019dbe2b7e",
|
| 422 |
+
"utils/misc.py": "be4a323116a0956a77960d1baf613f268057d6a36023663d2af9f8e4728cbe77",
|
| 423 |
+
"utils/plot_levels.py": "916b39885ced191960c26c334a3b16f9c99751f6c29c46bd69d14fee414965f3",
|
| 424 |
+
"utils/registry.py": "f7a44c6d18433100873d5df462b9f08a8bdeb11cf3d92226334c7e5ece318318",
|
| 425 |
+
"utils/train.py": "43eef988000f3a5e20856653d725337a0fe0dbe276f0cf80e1b200292abe812d",
|
| 426 |
+
"train_iclr_debug.py": "94844fceb0f0d1e0e48fa27384887ef727f11ce820ebdc4805e58982434e0d35",
|
| 427 |
+
"train_joint.py": "21b7c4f179632e9c7e319178e4d661f01a3702215dcbe1c8aeae90e430ebb4bf",
|
| 428 |
+
"config/iclr-debug/HF-MODEL-CARD.md": "de1fe176ebde262eacdf32af7d3f8c6497a764dd98e5a9094dad26de4aade38e",
|
| 429 |
+
"config/iclr-debug/full-owt-audit.json": "fefa394834f6e9f65944d3b514633c29061071ebe6ca60fce21a6edef9748c1e",
|
| 430 |
+
"config/iclr-debug/VANILLA.md": "29f86c602b4aaf5caf42e2397346c19c0e99c5f29911be117c59eb134f99114b",
|
| 431 |
+
"config/iclr-debug/MEDIUM.md": "16cdf56a3fb9f4b29ff1c05b49976f941cd630527094d363906e3e55b815d1c9",
|
| 432 |
+
"config/iclr-debug/benchmark-16sq.yaml": "18c9cef7032f9d3806d8b0c8b20fe0fc7d1ea98ad1d2440e24a67ee873c97f9b",
|
| 433 |
+
"config/iclr-debug/NCM-BUDGET-AUDIT.md": "e734caeab7edb612de7ff508a8c0ec52445530ab7a35895b5b793d6a7b752e4b",
|
| 434 |
+
"config/iclr-debug/medium.yaml": "1157a2c8ab87ae107ba634f3a11f186dc7f8eef02ff26a0a317be0e1fe6d71c6",
|
| 435 |
+
"config/iclr-debug/README.md": "1c757d2ff37026d08c22324ec44cfa760b0599af62c8ad9b728ada471a8f2905",
|
| 436 |
+
"config/iclr-debug/training-defaults.yaml": "c9c712942ab33cfdc8787363219424dd7d7f6505d5c6a3795b0ab0295b7a508a",
|
| 437 |
+
"config/iclr-debug/BENCHMARK.md": "2781ea506212e12a0640a4b0d2a2c881355ec85756ca4014bccb1b1aedccdf59",
|
| 438 |
+
"config/iclr-debug/TRAINING.md": "5611186130981f69cec45797289f392a27922bee764348fbbc4ce525e66760c8"
|
| 439 |
+
},
|
| 440 |
+
"dataset": {
|
| 441 |
+
"dataset": "hazyresearch/ncm-tokenized-datasets",
|
| 442 |
+
"revision": "951e04517fd53d57521d8e09abd6511a662787d0",
|
| 443 |
+
"dtype": "little-endian uint16",
|
| 444 |
+
"eos_id": 50256,
|
| 445 |
+
"offset_units": "token positions; [0, each EOS position + 1] with final sentinel",
|
| 446 |
+
"sampling": "length-weighted within-segment windows; pad short segments with 50257",
|
| 447 |
+
"caveat": "EOS segments are not independently verified source-document identities",
|
| 448 |
+
"splits": {
|
| 449 |
+
"train": {
|
| 450 |
+
"bytes": 18070847356,
|
| 451 |
+
"tokens": 9035423678,
|
| 452 |
+
"sha256": "b24b78a07f56974fd0c9855dc16f25b836d2cb172be23eb5d3c650cd6d73f56a",
|
| 453 |
+
"min_token_id": 0,
|
| 454 |
+
"max_token_id": 50256,
|
| 455 |
+
"segments": 8009770,
|
| 456 |
+
"trailing_tokens": 0,
|
| 457 |
+
"eos_only_segments": 0,
|
| 458 |
+
"length_percentiles": {
|
| 459 |
+
"min": 132.0,
|
| 460 |
+
"p25": 416.0,
|
| 461 |
+
"p50": 718.0,
|
| 462 |
+
"p75": 1249.0,
|
| 463 |
+
"p99": 7610.0,
|
| 464 |
+
"max": 131288.0
|
| 465 |
+
},
|
| 466 |
+
"segments_shorter_than_256": 673011,
|
| 467 |
+
"expected_padding_fraction_at_256": 1.5955357567300495e-05,
|
| 468 |
+
"offsets_status": "created",
|
| 469 |
+
"offsets_sha256": "7e84564946b8cf8190dbdc1e86bdf2fe8cfd9235a517328c80b4df038ea0028b",
|
| 470 |
+
"offset_entries": 8009771
|
| 471 |
+
},
|
| 472 |
+
"valid": {
|
| 473 |
+
"bytes": 9186834,
|
| 474 |
+
"tokens": 4593417,
|
| 475 |
+
"sha256": "1f2ee17e08327a39d4e2417948d02c7b998d626d2dcb2661197a6e208f58266f",
|
| 476 |
+
"min_token_id": 0,
|
| 477 |
+
"max_token_id": 50256,
|
| 478 |
+
"segments": 3999,
|
| 479 |
+
"trailing_tokens": 0,
|
| 480 |
+
"eos_only_segments": 0,
|
| 481 |
+
"length_percentiles": {
|
| 482 |
+
"min": 143.0,
|
| 483 |
+
"p25": 433.0,
|
| 484 |
+
"p50": 734.0,
|
| 485 |
+
"p75": 1292.5,
|
| 486 |
+
"p99": 7122.719999999997,
|
| 487 |
+
"max": 24868.0
|
| 488 |
+
},
|
| 489 |
+
"segments_shorter_than_256": 363,
|
| 490 |
+
"expected_padding_fraction_at_256": 1.6064002171981108e-05,
|
| 491 |
+
"offsets_status": "created",
|
| 492 |
+
"offsets_sha256": "09a748ce1dcec7de36c77c48e39ccbf5c0293cc073689fa7c5f48bdfb9bb9445",
|
| 493 |
+
"offset_entries": 4000
|
| 494 |
+
}
|
| 495 |
+
},
|
| 496 |
+
"overlap": {
|
| 497 |
+
"exact_train_segment_matches": 0,
|
| 498 |
+
"unique_validation_segments_in_train": 0,
|
| 499 |
+
"unique_validation_segments": 3999,
|
| 500 |
+
"near_duplicates_checked": false
|
| 501 |
+
}
|
| 502 |
+
},
|
| 503 |
+
"hf_repo": "iskhare/iclr-debug",
|
| 504 |
+
"generation_evaluation": true,
|
| 505 |
+
"versions": {
|
| 506 |
+
"torch": "2.13.0+cu130",
|
| 507 |
+
"transformers": "4.44.2",
|
| 508 |
+
"numpy": "1.26.4",
|
| 509 |
+
"hydra-core": "1.3.7"
|
| 510 |
+
}
|
| 511 |
+
}
|
iclr-debug-vanilla-joint-135b-v2/vanilla-joint/config.yaml
ADDED
|
@@ -0,0 +1,219 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
task: joint-next-concept
|
| 2 |
+
experiment: iclr-debug-vanilla-joint-135b-v2-vanilla-joint
|
| 3 |
+
experiment_dir: /home/ishank/mstok-results/iclr-debug-vanilla-joint-135b-v2/vanilla-joint
|
| 4 |
+
dataset:
|
| 5 |
+
train_source: /home/ishank/data/full_owt/train_gpt2.bin
|
| 6 |
+
validate_source: /home/ishank/data/full_owt/valid_gpt2.bin
|
| 7 |
+
pad_token_id: 50257
|
| 8 |
+
pre_tokenizer:
|
| 9 |
+
name: hf
|
| 10 |
+
tokenizer: hf
|
| 11 |
+
model_id: gpt2
|
| 12 |
+
special_tokens:
|
| 13 |
+
eos_token: <|endoftext|>
|
| 14 |
+
pad_token: <|pad|>
|
| 15 |
+
codec:
|
| 16 |
+
n_layers: 6
|
| 17 |
+
context_length: 256
|
| 18 |
+
embed_dim: 384
|
| 19 |
+
in_vocab_size: 50304
|
| 20 |
+
vqvae_vocab_size:
|
| 21 |
+
- 16384
|
| 22 |
+
- 16384
|
| 23 |
+
- 16384
|
| 24 |
+
- 16384
|
| 25 |
+
- 16384
|
| 26 |
+
- 16384
|
| 27 |
+
- 16384
|
| 28 |
+
- 16384
|
| 29 |
+
- 16384
|
| 30 |
+
- 16384
|
| 31 |
+
- 16384
|
| 32 |
+
- 16384
|
| 33 |
+
- 16384
|
| 34 |
+
- 16384
|
| 35 |
+
- 16384
|
| 36 |
+
- 16384
|
| 37 |
+
compression_factor: 1
|
| 38 |
+
dropout: 0.1
|
| 39 |
+
pre_quant_groupnorm: 4
|
| 40 |
+
pre_quant_dropout: 0.2
|
| 41 |
+
vector_quantizer_config:
|
| 42 |
+
clss: multiscale_residual_vector_quantizer
|
| 43 |
+
decay: 0.99
|
| 44 |
+
epsilon: 1.0e-05
|
| 45 |
+
commitment_cost: 0.25
|
| 46 |
+
learned_l1_sampling: true
|
| 47 |
+
learned_all_sampling: false
|
| 48 |
+
quant_resi:
|
| 49 |
+
enabled: true
|
| 50 |
+
ratio: 0.5
|
| 51 |
+
share_mode: 0
|
| 52 |
+
learnable_ratio: false
|
| 53 |
+
levels:
|
| 54 |
+
use_manual_levels: true
|
| 55 |
+
manual_levels:
|
| 56 |
+
- 1
|
| 57 |
+
- 4
|
| 58 |
+
- 9
|
| 59 |
+
- 16
|
| 60 |
+
- 25
|
| 61 |
+
- 36
|
| 62 |
+
- 49
|
| 63 |
+
- 64
|
| 64 |
+
- 81
|
| 65 |
+
- 100
|
| 66 |
+
- 121
|
| 67 |
+
- 144
|
| 68 |
+
- 169
|
| 69 |
+
- 196
|
| 70 |
+
- 225
|
| 71 |
+
- 256
|
| 72 |
+
aux:
|
| 73 |
+
fine_drop:
|
| 74 |
+
prob: 0.5
|
| 75 |
+
min_keep: 1
|
| 76 |
+
generator:
|
| 77 |
+
n_layer: 12
|
| 78 |
+
n_head: 12
|
| 79 |
+
bias: true
|
| 80 |
+
dropout: 0.1
|
| 81 |
+
n_embd: 768
|
| 82 |
+
context_length: 256
|
| 83 |
+
vocab_size:
|
| 84 |
+
- 16384
|
| 85 |
+
- 16384
|
| 86 |
+
- 16384
|
| 87 |
+
- 16384
|
| 88 |
+
- 16384
|
| 89 |
+
- 16384
|
| 90 |
+
- 16384
|
| 91 |
+
- 16384
|
| 92 |
+
- 16384
|
| 93 |
+
- 16384
|
| 94 |
+
- 16384
|
| 95 |
+
- 16384
|
| 96 |
+
- 16384
|
| 97 |
+
- 16384
|
| 98 |
+
- 16384
|
| 99 |
+
attn_pattern: block_diagonal
|
| 100 |
+
use_positional_encoding: true
|
| 101 |
+
use_level_encoding: true
|
| 102 |
+
prefix_len: 0
|
| 103 |
+
use_rope: true
|
| 104 |
+
rope_base: 10000
|
| 105 |
+
use_qk_norm: true
|
| 106 |
+
post_upsample_conv:
|
| 107 |
+
enabled: true
|
| 108 |
+
kernel_size: 3
|
| 109 |
+
use_relu2: true
|
| 110 |
+
use_flex_attention: true
|
| 111 |
+
shared_output_head: true
|
| 112 |
+
shared_head_per_level_bias: false
|
| 113 |
+
shared_head_per_level_scale: false
|
| 114 |
+
shared_head_adapter_rank: 0
|
| 115 |
+
single_shared_projection: true
|
| 116 |
+
objectives:
|
| 117 |
+
codec_weight: 1.0
|
| 118 |
+
ncp_weight: 1.0
|
| 119 |
+
soft_assignment_temperature: 1.0
|
| 120 |
+
optimization:
|
| 121 |
+
codec_lr: 0.001
|
| 122 |
+
codec_min_lr: 0.0001
|
| 123 |
+
codec_warmup_iters: 0
|
| 124 |
+
codec_lr_decay_iters: 171662
|
| 125 |
+
generator_lr: 0.0005
|
| 126 |
+
generator_min_lr: 1.0e-05
|
| 127 |
+
generator_warmup_iters: 300
|
| 128 |
+
generator_lr_decay_iters: 171662
|
| 129 |
+
beta_1: 0.9
|
| 130 |
+
codec_beta_2: 0.99
|
| 131 |
+
generator_beta_2: 0.99
|
| 132 |
+
weight_decay: 0.1
|
| 133 |
+
max_grad_norm: 1.0
|
| 134 |
+
level_loss_alpha: 1.0
|
| 135 |
+
grad_accumulation_steps: 6
|
| 136 |
+
corruption:
|
| 137 |
+
mode: per_level
|
| 138 |
+
per_level_probs:
|
| 139 |
+
- 0.85
|
| 140 |
+
- 0.8321428571
|
| 141 |
+
- 0.8142857143
|
| 142 |
+
- 0.7964285714
|
| 143 |
+
- 0.7785714286
|
| 144 |
+
- 0.7607142857
|
| 145 |
+
- 0.7428571429
|
| 146 |
+
- 0.725
|
| 147 |
+
- 0.7071428571
|
| 148 |
+
- 0.6892857143
|
| 149 |
+
- 0.6714285714
|
| 150 |
+
- 0.6535714286
|
| 151 |
+
- 0.6357142857
|
| 152 |
+
- 0.6178571429
|
| 153 |
+
- 0.6
|
| 154 |
+
skip_level0: true
|
| 155 |
+
training:
|
| 156 |
+
log_interval: 10
|
| 157 |
+
eval_interval: 1000
|
| 158 |
+
checkpoint_interval: 1000
|
| 159 |
+
eval_batch_size: 4
|
| 160 |
+
val_iters: 25
|
| 161 |
+
keep_last: 3
|
| 162 |
+
seed: 55
|
| 163 |
+
codec_initialization_seed: 42
|
| 164 |
+
generator_initialization_seed: 55
|
| 165 |
+
total_iters: 171662
|
| 166 |
+
batch_size: 64
|
| 167 |
+
expected_world_size: 8
|
| 168 |
+
expected_global_batch_size: 3072
|
| 169 |
+
resume_checkpoint: null
|
| 170 |
+
milestone_steps:
|
| 171 |
+
- 1272
|
| 172 |
+
- 6358
|
| 173 |
+
- 8000
|
| 174 |
+
- 17167
|
| 175 |
+
- 34333
|
| 176 |
+
- 68665
|
| 177 |
+
- 102997
|
| 178 |
+
- 137330
|
| 179 |
+
- 171662
|
| 180 |
+
generation_eval_steps:
|
| 181 |
+
- 8000
|
| 182 |
+
- 17167
|
| 183 |
+
- 34333
|
| 184 |
+
- 68665
|
| 185 |
+
- 102997
|
| 186 |
+
- 137330
|
| 187 |
+
- 171662
|
| 188 |
+
torch_compile:
|
| 189 |
+
enable: true
|
| 190 |
+
scope: loss
|
| 191 |
+
mode: default
|
| 192 |
+
dynamic: false
|
| 193 |
+
fullgraph: true
|
| 194 |
+
backend: inductor
|
| 195 |
+
mixed_precision:
|
| 196 |
+
enable: true
|
| 197 |
+
wandb:
|
| 198 |
+
entity: mstok
|
| 199 |
+
project: iclr-debug
|
| 200 |
+
group: full-owt-ctx256-16sq-135b-v1
|
| 201 |
+
enable: true
|
| 202 |
+
id: iclr-debug-vanilla-joint-135b-v2-vanilla-joint
|
| 203 |
+
resume: allow
|
| 204 |
+
gradients_and_params:
|
| 205 |
+
enable: false
|
| 206 |
+
log: all
|
| 207 |
+
log_freq: 1000
|
| 208 |
+
export:
|
| 209 |
+
vqvae_config_template: /home/ubuntu/mstok/config/repro-ctx256/vqvae.yaml
|
| 210 |
+
ncp_config_template: /home/ubuntu/mstok/config/repro-ctx256/ncp-sharedhead.yaml
|
| 211 |
+
iclr_debug:
|
| 212 |
+
stage: vanilla-joint
|
| 213 |
+
target_positions: 135000000000
|
| 214 |
+
lr_horizon_positions: 135000000000
|
| 215 |
+
stop_step: 171662
|
| 216 |
+
pilot: false
|
| 217 |
+
tokenizer_checkpoint: null
|
| 218 |
+
tokenizer_steps: null
|
| 219 |
+
budget_unit: input positions including padding; log non-padding tokens separately
|
iclr-debug-vanilla-joint-135b-v2/vanilla-joint/exports/step-1272/ncp.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:02a793091039dc52d5923fa82c5bb7cbab680125a33305740703d9ae648d4fc1
|
| 3 |
+
size 427469854
|
iclr-debug-vanilla-joint-135b-v2/vanilla-joint/exports/step-1272/vqvae.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:3a594c18d708c03245090ec0a6403ace9730d9969cf782eee452a246aeb2afd5
|
| 3 |
+
size 1300864085
|
iclr-debug-vanilla-joint-135b-v2/vanilla-joint/milestone-iter-1272.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:5d92e08b431f11c31a9d7ffbf9f64bcd0a4ac80f4995789665e4928020cec3f9
|
| 3 |
+
size 3550247001
|