File size: 1,668 Bytes
80300e5 b41d6e3 80300e5 b41d6e3 80300e5 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 | {
"schema": "exllm-training-release-v1",
"model": "EXLLM-0.005B-Instruct",
"checkpoint": "weights/EXLLM-v1.1-5m-release3.pt",
"checkpoint_sha256": "48ad00c6640689595a253fc949cad83f3000e14cccdf69275fcf245e85be3ceb",
"parameters": 5377824,
"parameters_billions": 0.005377824,
"global_step": 1810,
"lineage": {
"project_origin": "random initialization",
"release_parent": "weights/EXLLM-v1.0.0.safetensors",
"release_parent_sha256": "caad6a93aa2a8f6f2b525c4671dc8d2f8e4328d262a093172a55c672cd4e24c9",
"expansion": "dimension-compatible parameter transplantation",
"external_pretrained_checkpoint": null,
"stage_manifest": "training/release-5m-stages.json"
},
"architecture": {
"vocab_size": 868,
"max_seq_len": 128,
"d_model": 288,
"n_layers": 6,
"n_heads": 9,
"d_ff": 896,
"activation": "relu",
"normalization": "pre-norm RMSNorm",
"rms_eps": 0.00001,
"position_encoding": "learned absolute"
},
"base_training": {
"optimizer": "AdamW",
"learning_rate": 0.0005,
"betas": [0.9, 0.95],
"weight_decay": 0.03,
"gradient_clip_norm": 1.0,
"schedule": "linear warm-up and cosine decay"
},
"final_corrective_stage": {
"source": "weights/EXLLM-v1.1-5m-release2.pt",
"data": "data/v1_1_release_fix.jsonl",
"steps": 100,
"learning_rate": 0.000006,
"loss_first": 0.038745223909500054,
"loss_last": 0.024198499539634214
},
"published_staged_data": {
"jsonl_file_records": 86408,
"records_millions": 0.086408,
"deduplicated": false,
"note": "Sum of records across overlapping staged files; not a unique-example count."
}
}
|