| { |
| "checkpoint_sha256": "c4e8b5fb5730242d790d469806a8735be2424b9351dcdc25f643980ba3028236", |
| "weights_sha256": "05e2a496967bdd36da3e1bdd4fb8b10277b92a319b1f5f1e61062454eecc7b2e", |
| "parameters": 148910738, |
| "reference_step": 76294, |
| "optimizer_updates": 39147, |
| "continuation_tokens": 20000014336, |
| "provenance": { |
| "source_repo": "SlayerLab/gollem-v5-ckpts", |
| "source_revision": "963440ce6ab4ada7e95da4c1faa28ebb00c082d4", |
| "source_file": "final_128m_16x768/ckpt_760k.pt", |
| "source_sha256": "95d8b43fcd9c5fa06566da13dd03856b00300bad6f37fb4dc9442c18be9ac372", |
| "source_steps": 760000, |
| "source_training_tokens": 24903680000, |
| "parameters": 148910738, |
| "checks": [ |
| { |
| "length": 16, |
| "max_absolute_logit_error": 0.0 |
| }, |
| { |
| "length": 128, |
| "max_absolute_logit_error": 0.0 |
| }, |
| { |
| "length": 256, |
| "max_absolute_logit_error": 0.0 |
| } |
| ], |
| "optimizer": "reset for continued pretraining", |
| "method": "FFN expansion with zero new down columns; append three identity residual blocks" |
| }, |
| "safetensors_roundtrip": "all model tensors exactly equal; tied weights restored", |
| "training_config": { |
| "model": { |
| "vocab": 12288, |
| "n_layer": 19, |
| "n_embd": 768, |
| "n_head": 12, |
| "block": 1024, |
| "ffn_mult": 2.8125, |
| "norm": "rmsnorm", |
| "norm_eps": 1e-06, |
| "pos": "rope", |
| "rope_theta": 100000.0, |
| "ffn": "swiglu", |
| "value_residual": true, |
| "qk_norm": true, |
| "logit_cap": 0.0, |
| "z_loss": 0.0 |
| }, |
| "train": { |
| "seed": 1337, |
| "micro_batch": 32, |
| "global_batch": 256, |
| "steps": 76294, |
| "warmup": 250, |
| "lr": 0.0002, |
| "min_lr": 2e-05, |
| "muon_lr": 0.006666666666666667, |
| "checkpoint_every": 1000, |
| "validate_every": 1000, |
| "log_every": 20, |
| "compile": true |
| }, |
| "data": { |
| "train": "/data/final/train.bin", |
| "validation": "/data/final/val.bin", |
| "sha256": "ecfd0a4040a0ece6f07f728f092a1b373b6d253b4adfbe259c5107c1fc16681d" |
| } |
| } |
| } |
