| { |
| "preset": "full", |
| "mode": "train", |
| "hf": { |
| "token": "<redacted>", |
| "output_repo_id": "CyanMonkey/Test-Subject-3", |
| "private": false, |
| "push": true, |
| "resume_from_hub": true, |
| "auto_map": true |
| }, |
| "data": { |
| "source": "hub", |
| "dataset_repo_id": "CyanMonkey/NextGen-1-32-1024", |
| "train_split": "train", |
| "val_split": null, |
| "val_rows": 2048, |
| "max_train_rows": null, |
| "tokenizer_id": "AxiomicLabs/GPT-X2.5-135M", |
| "doc_masking": "auto", |
| "shuffle_block_rows": 192, |
| "prefetch": 6, |
| "seed": 42 |
| }, |
| "model": { |
| "arch_name": "OleanderForCausalLM", |
| "vocab_size": 32770, |
| "hidden_size": 576, |
| "num_groups": 7, |
| "group_pattern": [ |
| "gdn2", |
| "gdn2", |
| "gdn2", |
| "gqa" |
| ], |
| "mlp_hidden": 1600, |
| "max_seq_len": 1024, |
| "norm_eps": 1e-06, |
| "init_std": 0.02, |
| "tie_embeddings": true, |
| "attn_res": true, |
| "residual_dtype": "bfloat16", |
| "gdn2": { |
| "heads": 9, |
| "head_dim": 64, |
| "conv_size": 4, |
| "norm_eps": 1e-05, |
| "allow_neg_eigval": false, |
| "gate_in_kernel": false |
| }, |
| "gqa": { |
| "q_heads": 9, |
| "kv_heads": 3, |
| "head_dim": 64, |
| "rotary_dim": 32, |
| "rope_theta": 100000.0, |
| "qk_norm": true, |
| "value_residual": true |
| } |
| }, |
| "train": { |
| "micro_batch": 32, |
| "grad_accum_steps": 16, |
| "max_steps": 28820, |
| "grad_clip": 1.0, |
| "seed": 42, |
| "dtype": "auto" |
| }, |
| "optim": { |
| "muon_variant": "muoneq_r", |
| "muon_lr": 0.02, |
| "muon_momentum": 0.95, |
| "muon_nesterov": true, |
| "muon_ns_steps": 5, |
| "muon_wd": 0.01, |
| "skinny_to_adamw": false, |
| "adamw_lr": 0.0015, |
| "adamw_betas": [ |
| 0.9, |
| 0.95 |
| ], |
| "adamw_eps": 1e-08, |
| "adamw_wd": 0.0 |
| }, |
| "schedule": { |
| "warmup_steps": 100, |
| "stable_until": 0.825, |
| "decay_shape": "1-sqrt", |
| "min_lr_ratio": 0.0 |
| }, |
| "tst": { |
| "enabled": true, |
| "phase_fraction": 0.3, |
| "bag_size": 6, |
| "alog_shift_at_switch": false, |
| "reset_opt_at_switch": false |
| }, |
| "loss": { |
| "chunk_tokens": 8192, |
| "z_loss": 0.0 |
| }, |
| "compile": { |
| "enabled": true, |
| "mode": "default", |
| "precompile": true, |
| "fallback_to_eager": true |
| }, |
| "generation": { |
| "enabled": true, |
| "every_n_steps": 1000, |
| "prompt": "Once upon a time", |
| "max_new_tokens": 64, |
| "temperature": 0.8, |
| "top_k": 50 |
| }, |
| "save": { |
| "work_dir": "./oleander_run", |
| "log_steps": 20, |
| "eval_steps": 500, |
| "eval_batches": 64, |
| "save_steps": 500, |
| "keep_last_n": 2, |
| "push_steps": 500, |
| "resume_push_steps": 500, |
| "export_dtype": "float32", |
| "cpt_checkpoint": true |
| }, |
| "export": { |
| "model_name": "Oleander-150M", |
| "license": "apache-2.0", |
| "write_modeling_files": true, |
| "write_tokenizer_files": true, |
| "write_readme": true, |
| "write_training_meta": true, |
| "write_generation_config": true, |
| "meta_extra": {}, |
| "tokenizer_files": [ |
| "tokenizer.json", |
| "tokenizer_config.json", |
| "special_tokens_map.json", |
| "vocab.json", |
| "merges.txt", |
| "added_tokens.json", |
| "chat_template.jinja", |
| "tokenizer.model" |
| ] |
| }, |
| "bench": { |
| "micro_batches": [ |
| 16, |
| 32 |
| ], |
| "iters": 8, |
| "warmup": 3, |
| "profile": true |
| }, |
| "diloco": { |
| "enabled": false, |
| "is_leader": true, |
| "worker_id": 0, |
| "n_workers": 2, |
| "inner_steps": 500, |
| "outer_lr": 0.7, |
| "outer_momentum": 0.9, |
| "outer_nesterov": true, |
| "delta_dtype": "bfloat16", |
| "min_workers": 2, |
| "quorum_grace_s": 600, |
| "poll_s": 20, |
| "outlier_factor": 3.0, |
| "support_repo_id": "CyanMonkey/TestSubject-3-support", |
| "transport": "hub", |
| "local_dir": "./oleander_dl_local", |
| "reset_support": false, |
| "force_leader": false, |
| "lease_ttl_s": 900, |
| "lease_beat_s": 300, |
| "keep_rounds": 2, |
| "outer_state_rounds": 4, |
| "opt_snapshot_rounds": 4, |
| "ramp_steps": 50, |
| "og_push_rounds": 1, |
| "sample_every_rounds": 5, |
| "squash_every_rounds": 25 |
| } |
| } |
