File size: 727 Bytes
ea51a3d | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 | {
"architecture": "causal_transwave_blocklifting_meyer_byte_lm_g4_v2",
"source_commit": "5cae31b81e15160379ded79f32bcbb443d7db9f7",
"parameters": 96130040,
"dataset": "allenai/c4:en",
"objective": "exact causal next-byte NLL",
"context_tokens": 20480,
"physical_batch": 280,
"tokens_per_full_step": 5734400,
"maximum_training_tokens": 4499996672,
"usable_training_tokens": 4499988480,
"target_ppl": 3.0,
"base_lr": 0.0001,
"min_lr": 1e-05,
"warmup_fraction": 0.01,
"grad_clip": 1.0,
"model_seed": 20261010,
"order_seed": 20261011,
"allocator": "expandable_segments:True,garbage_collection_threshold:0.80",
"fingerprint": "370d05a6d2672fc7838726e78e8fe267b4b9c480b4f3708b64e7136beea128fd"
} |