{ "run_name": "cd140m-n5v2-dt0p3-mr27-mbd8-100k", "out_dir": "/datastor2/abao/checkpoints/parcae_runs/cd140m-n5v2-dt0p3-mr27-mbd8-100k", "resume": false, "resume_checkpoint_path": null, "max_tokens": 0, "max_steps": 100000, "max_flops": null, "token_to_param_ratio": null, "exit_at_step": null, "seed": 42, "model_name": "parcae-small-140m", "model_impl": "dynamic", "block_size": 29, "ignore_block_size_mismatch": true, "model_checkpoint": "/datastor2/abao/checkpoints/parcae_runs/init/parcae-140m-fabric.pt", "doc_block_attn": false, "cache_attn": false, "eod_token": null, "recurrence_schedule": "linear", "cirriculum_steps": 0, "world_batch_size": 512, "batch_size_ramp": 0, "optimizer": "MuonAdamW", "optim_config": { "lr": 0.005, "muon_lr": 0.005, "weight_decay": 0.0, "betas": [ 0.9, 0.95 ], "eps": 1e-10, "muon_momentum": 0.95, "muon_wd": 0.0, "muon_ns_steps": 5, "muon_momentum_warmup_steps": 100 }, "grad_clip": 1.0, "warmup_steps": 0.05, "cooldown_steps": 0.2, "lr_schedule": "trapezoid", "min_lr": 0.0, "no_weight_decay_for_bias_and_norm_params": true, "label_smoothing": 0.0, "z_regularization": 0.0, "goldfish": { "k_token_loss_dropout": 4, "start_position": 0, "context_width": 13, "strategy": null }, "fabric_strategy": "ddp", "fabric_precision": "bf16-mixed", "fabric_use_lightning_environment": false, "fabric": { "optimize_communication": false, "all_reduce_dtype": null, "row_tensor_parallel_size": 1, "col_tensor_parallel_size": 1, "depth_tensor_parallel_size": 1, "optim_sharding": false, "allow_optim_fusion": false, "use_apex_adamw": false, "fsdp_use_original_params": false }, "micro_batch_size": 512, "compile_model": true, "compile_backend": "inductor", "compile_mode": "default", "compile_fullgraph": false, "compile_dynamic": false, "matmul_precision": "high", "dataloader_num_workers": 0, "n_chunks": 4, "gradient_checkpointing": false, "allow_nonfinite_loss": false, "compiled_autograd": false, "compile_optimizer": false, "dynamo_ddp_config": "ddp_optimizer", "loss_guardrail_active": false, "skip_nonfinite_grads": true, "fail_instead_of_recompile": false, "logger_name": "wandb", "logger_entity": "gilpinlab", "logger_project": "parcae-countdown", "logger_group": "cd140m-n5v2-long", "data_telemetry": false, "model_telemetry": true, "extreme_telemetry": false, "shape_watching_steps": 3, "log_step_interval": 1, "eval_iters": 3, "eval_precision": "same", "save_step_interval": 10000, "eval_step_interval": 500, "save_first_step": false, "save_last_step": true, "save_n_min_before_job_done": null, "save_keep_last": 9, "best_checkpoint_metric": null, "best_checkpoint_mode": "max", "measure_utilization": true, "partial_depth_eval": [ 8, 27, 32 ], "generative_eval": { "enabled": true, "target": "countdown_store_eval_hook:evaluate", "depths": [ 8, 27, 32 ], "num_batches": 1, "seed": 1234, "max_new_tokens": null, "extra": { "eval_train": "/datastor2/abao/data/countdown/store_mid_n5v2_sft/eval_train", "eval_id": "/datastor2/abao/data/countdown/store_mid_n5v2_sft/eval_id", "eval_ood": "/datastor2/abao/data/countdown/store_mid_n5v2_sft/eval_ood" } }, "shuffle_filenames": true, "shuffle_blocks": true, "pad_to_block_size": true, "add_bos": true, "add_eos": true, "pack_sequences": false, "pack_buffer_size": 1000, "data_signature": { "keys": [ "text" ], "format_fn": "pass_text" }, "collate_checks_enabled": true, "all_block_size_tensors": false, "use_chat_template": false, "data_config": { "train_data": [ { "type": "hfds", "prefix": "countdown-train", "weight": 1, "data_signature": { "keys": [ "input", "target" ], "format_fn": "condition_input_supervise_target" }, "name": null, "data_dir": "/datastor2/abao/data/countdown/store_mid_n5v2_sft/train", "text_key": null, "repetitions": null, "wrap_epochs": true, "max_epoch": null, "scheduler": null, "return_data_id": false } ], "val_data": [ { "type": "hfds", "prefix": "countdown-val", "weight": 1, "data_signature": { "keys": [ "input", "target" ], "format_fn": "condition_input_supervise_target" }, "name": null, "data_dir": "/datastor2/abao/data/countdown/store_mid_n5v2_sft/val", "text_key": null, "repetitions": null, "wrap_epochs": false, "max_epoch": null, "scheduler": null, "return_data_id": false } ] }, "train_data_dir": "", "val_data_dir": "", "tokenizer_path": "/datastor2/abao/checkpoints/parcae_runs/cd140m-n5v2-dt0p3-mr27-mbd8-100k/tokenizer", "model_config": { "name": "parcae-small-140m", "hf_config": { "org": "SandyResearch", "name": "parcae-small-140m" }, "block_size": 2048, "n_embd": 768, "intermediate_size": 3072, "num_attention_heads": 6, "num_key_value_heads": 6, "vocab_size": 32768, "padding_multiple": 64, "padded_vocab_size": 32768, "rope_settings": { "use_rope": true, "rope_condense_ratio": 1, "rope_base": 50000 }, "use_abacus": false, "randomize_positions_from": null, "block_class_name": "TransformerPreNormBlock", "norm_class_name": "RMSNorm", "attn_impl": "flash", "norm_eps": 1e-05, "mlp_class_name": "BaseMLP", "nonlin_name": "ReLU2", "bias": false, "qk_bias": false, "init_strategy": "scaled-zero", "init_orthogonal": true, "skip_initialization": false, "mup_model_scaling_factor": 1, "use_fused_head": "pytorch", "debias_attention": false, "center_attention": false, "clip_qkv": null, "qk_norm": true, "logit_softcap": null, "activation_checkpoint_impl": "per-iteration", "simple_ops": false, "strategy": "ddp", "injection_type": "diagonal", "dt_scale": 0.3, "n_layers_in_recurrent_block": 2, "n_layers_in_prelude": 2, "n_layers_in_coda": 2, "state_init": "like-init", "recurrent_embedding_dimension": 768, "recurrent_intermediation_embedding_dimension": 3072, "recurrent_num_attention_heads": null, "prelude_norm": true, "sampling_scheme": "poisson-truncated-full", "mean_recurrence": 27, "mean_backprop_depth": 8, "lockstep_n": false, "lockstep_k": false, "curriculum_target": "forward", "recurrent_iteration_method": "per-sequence", "tie_embeddings": true, "model_class_name": "Parcae", "_is_recurrent_block_config": false }, "model_overwrite": { "vocab_size": 32768, "prelude_norm": true, "dt_scale": 0.3, "mean_recurrence": 27, "mean_backprop_depth": 8 } }