File size: 8,175 Bytes
eb259fd | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 | {
"run_name": "cd140m-n5v2-dt0p3-mr27-mbd8-100k",
"out_dir": "/datastor2/abao/checkpoints/parcae_runs/cd140m-n5v2-dt0p3-mr27-mbd8-100k",
"resume": false,
"resume_checkpoint_path": null,
"max_tokens": 0,
"max_steps": 100000,
"max_flops": null,
"token_to_param_ratio": null,
"exit_at_step": null,
"seed": 42,
"model_name": "parcae-small-140m",
"model_impl": "dynamic",
"block_size": 29,
"ignore_block_size_mismatch": true,
"model_checkpoint": "/datastor2/abao/checkpoints/parcae_runs/init/parcae-140m-fabric.pt",
"doc_block_attn": false,
"cache_attn": false,
"eod_token": null,
"recurrence_schedule": "linear",
"cirriculum_steps": 0,
"world_batch_size": 512,
"batch_size_ramp": 0,
"optimizer": "MuonAdamW",
"optim_config": {
"lr": 0.005,
"muon_lr": 0.005,
"weight_decay": 0.0,
"betas": [
0.9,
0.95
],
"eps": 1e-10,
"muon_momentum": 0.95,
"muon_wd": 0.0,
"muon_ns_steps": 5,
"muon_momentum_warmup_steps": 100
},
"grad_clip": 1.0,
"warmup_steps": 0.05,
"cooldown_steps": 0.2,
"lr_schedule": "trapezoid",
"min_lr": 0.0,
"no_weight_decay_for_bias_and_norm_params": true,
"label_smoothing": 0.0,
"z_regularization": 0.0,
"goldfish": {
"k_token_loss_dropout": 4,
"start_position": 0,
"context_width": 13,
"strategy": null
},
"fabric_strategy": "ddp",
"fabric_precision": "bf16-mixed",
"fabric_use_lightning_environment": false,
"fabric": {
"optimize_communication": false,
"all_reduce_dtype": null,
"row_tensor_parallel_size": 1,
"col_tensor_parallel_size": 1,
"depth_tensor_parallel_size": 1,
"optim_sharding": false,
"allow_optim_fusion": false,
"use_apex_adamw": false,
"fsdp_use_original_params": false
},
"micro_batch_size": 512,
"compile_model": true,
"compile_backend": "inductor",
"compile_mode": "default",
"compile_fullgraph": false,
"compile_dynamic": false,
"matmul_precision": "high",
"dataloader_num_workers": 0,
"n_chunks": 4,
"gradient_checkpointing": false,
"allow_nonfinite_loss": false,
"compiled_autograd": false,
"compile_optimizer": false,
"dynamo_ddp_config": "ddp_optimizer",
"loss_guardrail_active": false,
"skip_nonfinite_grads": true,
"fail_instead_of_recompile": false,
"logger_name": "wandb",
"logger_entity": "gilpinlab",
"logger_project": "parcae-countdown",
"logger_group": "cd140m-n5v2-long",
"data_telemetry": false,
"model_telemetry": true,
"extreme_telemetry": false,
"shape_watching_steps": 3,
"log_step_interval": 1,
"eval_iters": 3,
"eval_precision": "same",
"save_step_interval": 10000,
"eval_step_interval": 500,
"save_first_step": false,
"save_last_step": true,
"save_n_min_before_job_done": null,
"save_keep_last": 9,
"best_checkpoint_metric": null,
"best_checkpoint_mode": "max",
"measure_utilization": true,
"partial_depth_eval": [
8,
27,
32
],
"generative_eval": {
"enabled": true,
"target": "countdown_store_eval_hook:evaluate",
"depths": [
8,
27,
32
],
"num_batches": 1,
"seed": 1234,
"max_new_tokens": null,
"extra": {
"eval_train": "/datastor2/abao/data/countdown/store_mid_n5v2_sft/eval_train",
"eval_id": "/datastor2/abao/data/countdown/store_mid_n5v2_sft/eval_id",
"eval_ood": "/datastor2/abao/data/countdown/store_mid_n5v2_sft/eval_ood"
}
},
"shuffle_filenames": true,
"shuffle_blocks": true,
"pad_to_block_size": true,
"add_bos": true,
"add_eos": true,
"pack_sequences": false,
"pack_buffer_size": 1000,
"data_signature": {
"keys": [
"text"
],
"format_fn": "pass_text"
},
"collate_checks_enabled": true,
"all_block_size_tensors": false,
"use_chat_template": false,
"data_config": {
"train_data": [
{
"type": "hfds",
"prefix": "countdown-train",
"weight": 1,
"data_signature": {
"keys": [
"input",
"target"
],
"format_fn": "condition_input_supervise_target"
},
"name": null,
"data_dir": "/datastor2/abao/data/countdown/store_mid_n5v2_sft/train",
"text_key": null,
"repetitions": null,
"wrap_epochs": true,
"max_epoch": null,
"scheduler": null,
"return_data_id": false
}
],
"val_data": [
{
"type": "hfds",
"prefix": "countdown-val",
"weight": 1,
"data_signature": {
"keys": [
"input",
"target"
],
"format_fn": "condition_input_supervise_target"
},
"name": null,
"data_dir": "/datastor2/abao/data/countdown/store_mid_n5v2_sft/val",
"text_key": null,
"repetitions": null,
"wrap_epochs": false,
"max_epoch": null,
"scheduler": null,
"return_data_id": false
}
]
},
"train_data_dir": "",
"val_data_dir": "",
"tokenizer_path": "/datastor2/abao/checkpoints/parcae_runs/cd140m-n5v2-dt0p3-mr27-mbd8-100k/tokenizer",
"model_config": {
"name": "parcae-small-140m",
"hf_config": {
"org": "SandyResearch",
"name": "parcae-small-140m"
},
"block_size": 2048,
"n_embd": 768,
"intermediate_size": 3072,
"num_attention_heads": 6,
"num_key_value_heads": 6,
"vocab_size": 32768,
"padding_multiple": 64,
"padded_vocab_size": 32768,
"rope_settings": {
"use_rope": true,
"rope_condense_ratio": 1,
"rope_base": 50000
},
"use_abacus": false,
"randomize_positions_from": null,
"block_class_name": "TransformerPreNormBlock",
"norm_class_name": "RMSNorm",
"attn_impl": "flash",
"norm_eps": 1e-05,
"mlp_class_name": "BaseMLP",
"nonlin_name": "ReLU2",
"bias": false,
"qk_bias": false,
"init_strategy": "scaled-zero",
"init_orthogonal": true,
"skip_initialization": false,
"mup_model_scaling_factor": 1,
"use_fused_head": "pytorch",
"debias_attention": false,
"center_attention": false,
"clip_qkv": null,
"qk_norm": true,
"logit_softcap": null,
"activation_checkpoint_impl": "per-iteration",
"simple_ops": false,
"strategy": "ddp",
"injection_type": "diagonal",
"dt_scale": 0.3,
"n_layers_in_recurrent_block": 2,
"n_layers_in_prelude": 2,
"n_layers_in_coda": 2,
"state_init": "like-init",
"recurrent_embedding_dimension": 768,
"recurrent_intermediation_embedding_dimension": 3072,
"recurrent_num_attention_heads": null,
"prelude_norm": true,
"sampling_scheme": "poisson-truncated-full",
"mean_recurrence": 27,
"mean_backprop_depth": 8,
"lockstep_n": false,
"lockstep_k": false,
"curriculum_target": "forward",
"recurrent_iteration_method": "per-sequence",
"tie_embeddings": true,
"model_class_name": "Parcae",
"_is_recurrent_block_config": false
},
"model_overwrite": {
"vocab_size": 32768,
"prelude_norm": true,
"dt_scale": 0.3,
"mean_recurrence": 27,
"mean_backprop_depth": 8
}
} |