| { |
| "run_name": "cd140m-n5v2-dt0p3-mr27-mbd8-100k", |
| "out_dir": "/datastor2/abao/checkpoints/parcae_runs/cd140m-n5v2-dt0p3-mr27-mbd8-100k", |
| "resume": false, |
| "resume_checkpoint_path": null, |
| "max_tokens": 0, |
| "max_steps": 100000, |
| "max_flops": null, |
| "token_to_param_ratio": null, |
| "exit_at_step": null, |
| "seed": 42, |
| "model_name": "parcae-small-140m", |
| "model_impl": "dynamic", |
| "block_size": 29, |
| "ignore_block_size_mismatch": true, |
| "model_checkpoint": "/datastor2/abao/checkpoints/parcae_runs/init/parcae-140m-fabric.pt", |
| "doc_block_attn": false, |
| "cache_attn": false, |
| "eod_token": null, |
| "recurrence_schedule": "linear", |
| "cirriculum_steps": 0, |
| "world_batch_size": 512, |
| "batch_size_ramp": 0, |
| "optimizer": "MuonAdamW", |
| "optim_config": { |
| "lr": 0.005, |
| "muon_lr": 0.005, |
| "weight_decay": 0.0, |
| "betas": [ |
| 0.9, |
| 0.95 |
| ], |
| "eps": 1e-10, |
| "muon_momentum": 0.95, |
| "muon_wd": 0.0, |
| "muon_ns_steps": 5, |
| "muon_momentum_warmup_steps": 100 |
| }, |
| "grad_clip": 1.0, |
| "warmup_steps": 0.05, |
| "cooldown_steps": 0.2, |
| "lr_schedule": "trapezoid", |
| "min_lr": 0.0, |
| "no_weight_decay_for_bias_and_norm_params": true, |
| "label_smoothing": 0.0, |
| "z_regularization": 0.0, |
| "goldfish": { |
| "k_token_loss_dropout": 4, |
| "start_position": 0, |
| "context_width": 13, |
| "strategy": null |
| }, |
| "fabric_strategy": "ddp", |
| "fabric_precision": "bf16-mixed", |
| "fabric_use_lightning_environment": false, |
| "fabric": { |
| "optimize_communication": false, |
| "all_reduce_dtype": null, |
| "row_tensor_parallel_size": 1, |
| "col_tensor_parallel_size": 1, |
| "depth_tensor_parallel_size": 1, |
| "optim_sharding": false, |
| "allow_optim_fusion": false, |
| "use_apex_adamw": false, |
| "fsdp_use_original_params": false |
| }, |
| "micro_batch_size": 512, |
| "compile_model": true, |
| "compile_backend": "inductor", |
| "compile_mode": "default", |
| "compile_fullgraph": false, |
| "compile_dynamic": false, |
| "matmul_precision": "high", |
| "dataloader_num_workers": 0, |
| "n_chunks": 4, |
| "gradient_checkpointing": false, |
| "allow_nonfinite_loss": false, |
| "compiled_autograd": false, |
| "compile_optimizer": false, |
| "dynamo_ddp_config": "ddp_optimizer", |
| "loss_guardrail_active": false, |
| "skip_nonfinite_grads": true, |
| "fail_instead_of_recompile": false, |
| "logger_name": "wandb", |
| "logger_entity": "gilpinlab", |
| "logger_project": "parcae-countdown", |
| "logger_group": "cd140m-n5v2-long", |
| "data_telemetry": false, |
| "model_telemetry": true, |
| "extreme_telemetry": false, |
| "shape_watching_steps": 3, |
| "log_step_interval": 1, |
| "eval_iters": 3, |
| "eval_precision": "same", |
| "save_step_interval": 10000, |
| "eval_step_interval": 500, |
| "save_first_step": false, |
| "save_last_step": true, |
| "save_n_min_before_job_done": null, |
| "save_keep_last": 9, |
| "best_checkpoint_metric": null, |
| "best_checkpoint_mode": "max", |
| "measure_utilization": true, |
| "partial_depth_eval": [ |
| 8, |
| 27, |
| 32 |
| ], |
| "generative_eval": { |
| "enabled": true, |
| "target": "countdown_store_eval_hook:evaluate", |
| "depths": [ |
| 8, |
| 27, |
| 32 |
| ], |
| "num_batches": 1, |
| "seed": 1234, |
| "max_new_tokens": null, |
| "extra": { |
| "eval_train": "/datastor2/abao/data/countdown/store_mid_n5v2_sft/eval_train", |
| "eval_id": "/datastor2/abao/data/countdown/store_mid_n5v2_sft/eval_id", |
| "eval_ood": "/datastor2/abao/data/countdown/store_mid_n5v2_sft/eval_ood" |
| } |
| }, |
| "shuffle_filenames": true, |
| "shuffle_blocks": true, |
| "pad_to_block_size": true, |
| "add_bos": true, |
| "add_eos": true, |
| "pack_sequences": false, |
| "pack_buffer_size": 1000, |
| "data_signature": { |
| "keys": [ |
| "text" |
| ], |
| "format_fn": "pass_text" |
| }, |
| "collate_checks_enabled": true, |
| "all_block_size_tensors": false, |
| "use_chat_template": false, |
| "data_config": { |
| "train_data": [ |
| { |
| "type": "hfds", |
| "prefix": "countdown-train", |
| "weight": 1, |
| "data_signature": { |
| "keys": [ |
| "input", |
| "target" |
| ], |
| "format_fn": "condition_input_supervise_target" |
| }, |
| "name": null, |
| "data_dir": "/datastor2/abao/data/countdown/store_mid_n5v2_sft/train", |
| "text_key": null, |
| "repetitions": null, |
| "wrap_epochs": true, |
| "max_epoch": null, |
| "scheduler": null, |
| "return_data_id": false |
| } |
| ], |
| "val_data": [ |
| { |
| "type": "hfds", |
| "prefix": "countdown-val", |
| "weight": 1, |
| "data_signature": { |
| "keys": [ |
| "input", |
| "target" |
| ], |
| "format_fn": "condition_input_supervise_target" |
| }, |
| "name": null, |
| "data_dir": "/datastor2/abao/data/countdown/store_mid_n5v2_sft/val", |
| "text_key": null, |
| "repetitions": null, |
| "wrap_epochs": false, |
| "max_epoch": null, |
| "scheduler": null, |
| "return_data_id": false |
| } |
| ] |
| }, |
| "train_data_dir": "", |
| "val_data_dir": "", |
| "tokenizer_path": "/datastor2/abao/checkpoints/parcae_runs/cd140m-n5v2-dt0p3-mr27-mbd8-100k/tokenizer", |
| "model_config": { |
| "name": "parcae-small-140m", |
| "hf_config": { |
| "org": "SandyResearch", |
| "name": "parcae-small-140m" |
| }, |
| "block_size": 2048, |
| "n_embd": 768, |
| "intermediate_size": 3072, |
| "num_attention_heads": 6, |
| "num_key_value_heads": 6, |
| "vocab_size": 32768, |
| "padding_multiple": 64, |
| "padded_vocab_size": 32768, |
| "rope_settings": { |
| "use_rope": true, |
| "rope_condense_ratio": 1, |
| "rope_base": 50000 |
| }, |
| "use_abacus": false, |
| "randomize_positions_from": null, |
| "block_class_name": "TransformerPreNormBlock", |
| "norm_class_name": "RMSNorm", |
| "attn_impl": "flash", |
| "norm_eps": 1e-05, |
| "mlp_class_name": "BaseMLP", |
| "nonlin_name": "ReLU2", |
| "bias": false, |
| "qk_bias": false, |
| "init_strategy": "scaled-zero", |
| "init_orthogonal": true, |
| "skip_initialization": false, |
| "mup_model_scaling_factor": 1, |
| "use_fused_head": "pytorch", |
| "debias_attention": false, |
| "center_attention": false, |
| "clip_qkv": null, |
| "qk_norm": true, |
| "logit_softcap": null, |
| "activation_checkpoint_impl": "per-iteration", |
| "simple_ops": false, |
| "strategy": "ddp", |
| "injection_type": "diagonal", |
| "dt_scale": 0.3, |
| "n_layers_in_recurrent_block": 2, |
| "n_layers_in_prelude": 2, |
| "n_layers_in_coda": 2, |
| "state_init": "like-init", |
| "recurrent_embedding_dimension": 768, |
| "recurrent_intermediation_embedding_dimension": 3072, |
| "recurrent_num_attention_heads": null, |
| "prelude_norm": true, |
| "sampling_scheme": "poisson-truncated-full", |
| "mean_recurrence": 27, |
| "mean_backprop_depth": 8, |
| "lockstep_n": false, |
| "lockstep_k": false, |
| "curriculum_target": "forward", |
| "recurrent_iteration_method": "per-sequence", |
| "tie_embeddings": true, |
| "model_class_name": "Parcae", |
| "_is_recurrent_block_config": false |
| }, |
| "model_overwrite": { |
| "vocab_size": 32768, |
| "prelude_norm": true, |
| "dt_scale": 0.3, |
| "mean_recurrence": 27, |
| "mean_backprop_depth": 8 |
| } |
| } |
