| { |
| "model_type": "exp_train_hybrid", |
| "architectures": [ |
| "HybridLMForCausalLM" |
| ], |
| "auto_map": { |
| "AutoConfig": "patch.HybridLMConfig", |
| "AutoModelForCausalLM": "patch.HybridLMForCausalLM" |
| }, |
| "format": "exp_train checkpoint: model.safetensors (dense FP32 weights, tied embedding stored under both names), ngram.safetensors (BF16 table + FP32 row second moments), optim_rank*/state_rank* (.safetensors + .json structure; resume only), meta.json (step, tokens, checksums)", |
| "architecture": "hybrid LM: Gated DeltaNet + gated softmax attention, hashed n-gram memory", |
| "preset": "700m", |
| "ngram_params": 1000000000, |
| "vocab_size": 60800, |
| "bos_token_id": 2, |
| "eos_token_id": 3, |
| "pad_token_id": 0, |
| "max_doc_len": 900, |
| "torch_dtype": "float32", |
| "model_config": { |
| "vocab_size": 60800, |
| "d_model": 768, |
| "n_layers": 56, |
| "full_attn_every": 4, |
| "n_heads": 6, |
| "n_kv_heads": 2, |
| "head_dim": 128, |
| "rope_theta": 1000000.0, |
| "gdn_heads": 6, |
| "gdn_head_dim": 128, |
| "gdn_expand_v": 1.0, |
| "gdn_conv_size": 4, |
| "gdn_variant": "gdn", |
| "gdn2_gate_rank": 0, |
| "gdn2_lower_bound": 0.0, |
| "ffn_dim": 2304, |
| "norm_eps": 1e-06, |
| "gated_residual": true, |
| "tie_embeddings": false, |
| "ngram": { |
| "enabled": true, |
| "orders": [ |
| 2, |
| 3 |
| ], |
| "heads_per_order": 4, |
| "dim": 32, |
| "total_params": 1000000000, |
| "inject_layers": [ |
| 1, |
| 27 |
| ], |
| "table_dtype": "bfloat16", |
| "lr": 0.02, |
| "beta2": 0.99, |
| "eps": 1e-08, |
| "canonical_tokens": true, |
| "stochastic_round": true, |
| "sign_pool": 8192 |
| } |
| }, |
| "modifications": { |
| "delta_block": true, |
| "delta_blocks": 4, |
| "delta_variant": "repo", |
| "depth_attn": "off", |
| "ngram_canonical_sha256": "0e9f25836e7da68ad664bb0908c7acba962458643ca92a9e2f2d75c54e8ebb2c", |
| "layer_drop": 0.25, |
| "layer_drop_final": 0.05, |
| "gdn_variant": "gdn", |
| "tst_bag": 1, |
| "tst_steps": 0 |
| } |
| } |
|
|
