| { | |
| "model_type": "san", | |
| "vocab_size": 49152, | |
| "d_model": 384, | |
| "num_heads": 8, | |
| "num_kv_heads": 4, | |
| "num_layers": 12, | |
| "max_seq_len": 2048, | |
| "rope_theta": 100000.0, | |
| "dtype": "bfloat16", | |
| "engram_orders": [ | |
| 2, | |
| 3 | |
| ], | |
| "engram_heads": 0, | |
| "engram_slots": 8192, | |
| "engram_layers": [ | |
| 2, | |
| 15 | |
| ], | |
| "mhc_lanes": 4, | |
| "use_checkpoint": true, | |
| "ckpt_every": 1, | |
| "mtp_weight": 0.1, | |
| "tokenizer": "HuggingFaceTB/SmolLM2-135M", | |
| "trained_steps": 137260, | |
| "architecture": "Faithful PyTorch port of needle (arXiv:2607.18363). See san_model.py / GitHub kenpeter/x-small.", | |
| "note": "pytorch_model.bin = model_state_dict (load via san_model.SimpleAttentionNetwork.load_state_dict). san_latest.pt = raw training checkpoint (step 137260): {step, loss, model_state_dict, optimizer_state_dict, config}." | |
| } |