| { |
| "step": 17542, |
| "tokens_seen": 10346692608, |
| "raw_tokens_seen": 10346692608, |
| "tst": { |
| "bag": 1, |
| "steps": 0 |
| }, |
| "train_seconds": 547795.9212001314, |
| "ngram_canon_sha256": "0e9f25836e7da68ad664bb0908c7acba962458643ca92a9e2f2d75c54e8ebb2c", |
| "skipped_updates": 0, |
| "world_size": 1, |
| "preset": "700m", |
| "ngram_params": 1000000000, |
| "args": { |
| "preset": "700m", |
| "ngram_params": 1000000000, |
| "model_override": [ |
| "d_model=768", |
| "n_layers=56", |
| "n_heads=6", |
| "n_kv_heads=2", |
| "head_dim=128", |
| "gdn_heads=6", |
| "gdn_head_dim=128", |
| "ffn_dim=2304", |
| "ngram.heads_per_order=4", |
| "ngram.dim=32", |
| "ngram.inject_layers=1,27", |
| "ngram.canonical_tokens=True", |
| "ngram.stochastic_round=True", |
| "ngram.sign_pool=8192", |
| "tie_embeddings=False" |
| ], |
| "tokenizer": "/exp_train/tokenizer/tokenizer.json", |
| "data": "/exp_train/data/packed1-train.bin", |
| "eval_data": "/exp_train/data/packed1-val.bin", |
| "micro_tokens": 18432, |
| "seq_len": 900, |
| "grad_accum": 32, |
| "steps": 17542, |
| "stop_at": 0, |
| "warmup_steps": 88, |
| "decay_frac": 0.295, |
| "min_lr_frac": 0.0, |
| "tst_bag": 0, |
| "tst_frac": 0.2, |
| "optimizer": "dion3", |
| "dion_lr": 0.02, |
| "dion_fraction": 0.5, |
| "dion_mu": 0.95, |
| "dion_weight_decay": 0.01, |
| "dion_triton": true, |
| "dion_replicated": false, |
| "resync_every": 50, |
| "dion_compile": true, |
| "scalar_lr": 0.003, |
| "adamw_lr": 0.0003, |
| "grad_clip": 1.0, |
| "compile": "tail", |
| "compile_mode": "default", |
| "loss_chunks": 4, |
| "residual_dtype": "float32", |
| "fast_ce": true, |
| "update_threads": 80, |
| "ngram_sync": "default", |
| "ddp_comm_hook": "none", |
| "ddp_bucket_mb": 64, |
| "fla_nosync": true, |
| "layer_drop": 0.25, |
| "layer_drop_dist": "ild", |
| "layer_drop_schedule": "decreasing", |
| "layer_drop_final": 0.05, |
| "layer_drop_end": 0.705, |
| "depth_attn": "off", |
| "delta_block": true, |
| "delta_blocks": 4, |
| "delta_variant": "repo", |
| "eval_every": 3508, |
| "eval_tokens": 20000000, |
| "eval_exit_at": 42, |
| "ckpt_dir": "/exp_train/runs/packed1-500m-deep/ckpt", |
| "ckpt_every": 877, |
| "ckpt_minutes": 0.0, |
| "keep": 2, |
| "resume": "auto", |
| "max_minutes": 0.0, |
| "max_skips": 3, |
| "metrics": "/exp_train/runs/packed1-500m-deep/metrics.jsonl", |
| "output": "/exp_train/runs/packed1-500m-deep/results.jsonl", |
| "run_name": "packed1-500m-deep-a0", |
| "timing_skip": 2, |
| "peak_tflops": 0.0, |
| "gpu_mem_cap_gib": 59.0, |
| "seed": 1234, |
| "dist_backend": "nccl", |
| "profile_micro": 0, |
| "sync_debug": false, |
| "wandb_project": "exp_train", |
| "wandb_entity": "", |
| "wandb_name": "packed1-500m-deep", |
| "wandb_mode": "online", |
| "hub_repo": "Ba2han/packed1-500m-deep", |
| "hub_keep": 2, |
| "hub_private": true, |
| "hub_squash": true, |
| "table_probe_rows": 65536, |
| "eval_max_minutes": 30.0, |
| "eval_ngram_ablation": true, |
| "crash_at_step": -1, |
| "crash_point": "mid-update" |
| }, |
| "model_config": { |
| "vocab_size": 60800, |
| "d_model": 768, |
| "n_layers": 56, |
| "full_attn_every": 4, |
| "n_heads": 6, |
| "n_kv_heads": 2, |
| "head_dim": 128, |
| "rope_theta": 1000000.0, |
| "gdn_heads": 6, |
| "gdn_head_dim": 128, |
| "gdn_expand_v": 1.0, |
| "gdn_conv_size": 4, |
| "gdn_variant": "gdn", |
| "gdn2_gate_rank": 0, |
| "gdn2_lower_bound": 0.0, |
| "ffn_dim": 2304, |
| "norm_eps": 1e-06, |
| "gated_residual": true, |
| "tie_embeddings": false, |
| "ngram": { |
| "enabled": true, |
| "orders": [ |
| 2, |
| 3 |
| ], |
| "heads_per_order": 4, |
| "dim": 32, |
| "total_params": 1000000000, |
| "inject_layers": [ |
| 1, |
| 27 |
| ], |
| "table_dtype": "bfloat16", |
| "lr": 0.02, |
| "beta2": 0.99, |
| "eps": 1e-08, |
| "canonical_tokens": true, |
| "stochastic_round": true, |
| "sign_pool": 8192 |
| } |
| }, |
| "data_blocks": 561351, |
| "eval_batches": 1085, |
| "saved_at": "2026-11-02 23:41:11", |
| "checksums": { |
| "optim_rank0": 14332326.093579413, |
| "state_rank0": 318850.0, |
| "model": 45744.409796107095, |
| "ngram": -7987.134480161983 |
| }, |
| "format": "safetensors", |
| "files": { |
| "optim_rank0": [ |
| "optim_rank0.safetensors", |
| "optim_rank0.json" |
| ], |
| "state_rank0": [ |
| "state_rank0.safetensors", |
| "state_rank0.json" |
| ], |
| "model": [ |
| "model.safetensors" |
| ], |
| "ngram": [ |
| "ngram.safetensors" |
| ], |
| "assets": [ |
| "tokenizer.json", |
| "tokenizer_config.json", |
| "special_tokens_map.json", |
| "config.json" |
| ] |
| }, |
| "write_seconds": 11.22276174009312 |
| } |
