{
  "step": 17542,
  "tokens_seen": 10346692608,
  "raw_tokens_seen": 10346692608,
  "tst": {
    "bag": 1,
    "steps": 0
  },
  "train_seconds": 547795.9212001314,
  "ngram_canon_sha256": "0e9f25836e7da68ad664bb0908c7acba962458643ca92a9e2f2d75c54e8ebb2c",
  "skipped_updates": 0,
  "world_size": 1,
  "preset": "700m",
  "ngram_params": 1000000000,
  "args": {
    "preset": "700m",
    "ngram_params": 1000000000,
    "model_override": [
      "d_model=768",
      "n_layers=56",
      "n_heads=6",
      "n_kv_heads=2",
      "head_dim=128",
      "gdn_heads=6",
      "gdn_head_dim=128",
      "ffn_dim=2304",
      "ngram.heads_per_order=4",
      "ngram.dim=32",
      "ngram.inject_layers=1,27",
      "ngram.canonical_tokens=True",
      "ngram.stochastic_round=True",
      "ngram.sign_pool=8192",
      "tie_embeddings=False"
    ],
    "tokenizer": "/exp_train/tokenizer/tokenizer.json",
    "data": "/exp_train/data/packed1-train.bin",
    "eval_data": "/exp_train/data/packed1-val.bin",
    "micro_tokens": 18432,
    "seq_len": 900,
    "grad_accum": 32,
    "steps": 17542,
    "stop_at": 0,
    "warmup_steps": 88,
    "decay_frac": 0.295,
    "min_lr_frac": 0.0,
    "tst_bag": 0,
    "tst_frac": 0.2,
    "optimizer": "dion3",
    "dion_lr": 0.02,
    "dion_fraction": 0.5,
    "dion_mu": 0.95,
    "dion_weight_decay": 0.01,
    "dion_triton": true,
    "dion_replicated": false,
    "resync_every": 50,
    "dion_compile": true,
    "scalar_lr": 0.003,
    "adamw_lr": 0.0003,
    "grad_clip": 1.0,
    "compile": "tail",
    "compile_mode": "default",
    "loss_chunks": 4,
    "residual_dtype": "float32",
    "fast_ce": true,
    "update_threads": 80,
    "ngram_sync": "default",
    "ddp_comm_hook": "none",
    "ddp_bucket_mb": 64,
    "fla_nosync": true,
    "layer_drop": 0.25,
    "layer_drop_dist": "ild",
    "layer_drop_schedule": "decreasing",
    "layer_drop_final": 0.05,
    "layer_drop_end": 0.705,
    "depth_attn": "off",
    "delta_block": true,
    "delta_blocks": 4,
    "delta_variant": "repo",
    "eval_every": 3508,
    "eval_tokens": 20000000,
    "eval_exit_at": 42,
    "ckpt_dir": "/exp_train/runs/packed1-500m-deep/ckpt",
    "ckpt_every": 877,
    "ckpt_minutes": 0.0,
    "keep": 2,
    "resume": "auto",
    "max_minutes": 0.0,
    "max_skips": 3,
    "metrics": "/exp_train/runs/packed1-500m-deep/metrics.jsonl",
    "output": "/exp_train/runs/packed1-500m-deep/results.jsonl",
    "run_name": "packed1-500m-deep-a0",
    "timing_skip": 2,
    "peak_tflops": 0.0,
    "gpu_mem_cap_gib": 59.0,
    "seed": 1234,
    "dist_backend": "nccl",
    "profile_micro": 0,
    "sync_debug": false,
    "wandb_project": "exp_train",
    "wandb_entity": "",
    "wandb_name": "packed1-500m-deep",
    "wandb_mode": "online",
    "hub_repo": "Ba2han/packed1-500m-deep",
    "hub_keep": 2,
    "hub_private": true,
    "hub_squash": true,
    "table_probe_rows": 65536,
    "eval_max_minutes": 30.0,
    "eval_ngram_ablation": true,
    "crash_at_step": -1,
    "crash_point": "mid-update"
  },
  "model_config": {
    "vocab_size": 60800,
    "d_model": 768,
    "n_layers": 56,
    "full_attn_every": 4,
    "n_heads": 6,
    "n_kv_heads": 2,
    "head_dim": 128,
    "rope_theta": 1000000.0,
    "gdn_heads": 6,
    "gdn_head_dim": 128,
    "gdn_expand_v": 1.0,
    "gdn_conv_size": 4,
    "gdn_variant": "gdn",
    "gdn2_gate_rank": 0,
    "gdn2_lower_bound": 0.0,
    "ffn_dim": 2304,
    "norm_eps": 1e-06,
    "gated_residual": true,
    "tie_embeddings": false,
    "ngram": {
      "enabled": true,
      "orders": [
        2,
        3
      ],
      "heads_per_order": 4,
      "dim": 32,
      "total_params": 1000000000,
      "inject_layers": [
        1,
        27
      ],
      "table_dtype": "bfloat16",
      "lr": 0.02,
      "beta2": 0.99,
      "eps": 1e-08,
      "canonical_tokens": true,
      "stochastic_round": true,
      "sign_pool": 8192
    }
  },
  "data_blocks": 561351,
  "eval_batches": 1085,
  "saved_at": "2026-11-02 23:41:11",
  "checksums": {
    "optim_rank0": 14332326.093579413,
    "state_rank0": 318850.0,
    "model": 45744.409796107095,
    "ngram": -7987.134480161983
  },
  "format": "safetensors",
  "files": {
    "optim_rank0": [
      "optim_rank0.safetensors",
      "optim_rank0.json"
    ],
    "state_rank0": [
      "state_rank0.safetensors",
      "state_rank0.json"
    ],
    "model": [
      "model.safetensors"
    ],
    "ngram": [
      "ngram.safetensors"
    ],
    "assets": [
      "tokenizer.json",
      "tokenizer_config.json",
      "special_tokens_map.json",
      "config.json"
    ]
  },
  "write_seconds": 11.22276174009312
}