| { |
| "model_type": "juniper1", |
| "architecture": "j1-wide-20x768", |
| "freeze_id": "juniper1-base-v1", |
| "parameters": 142771456, |
| "torch_dtype": "float32", |
| "context_tokens_qualified": 2048, |
| "vocab_size": 24576, |
| "n_layer": 20, |
| "d_model": 768, |
| "n_head": 12, |
| "n_kv_head": 6, |
| "head_dim": 64, |
| "d_ff": 1920, |
| "max_seq_len": 2048, |
| "rope_theta": 100000.0, |
| "norm_eps": 1e-05, |
| "qk_norm": "rmsnorm_per_head", |
| "tie_embeddings": true, |
| "init_std": 0.02, |
| "attention": { |
| "type": "GQA", |
| "n_head": 12, |
| "n_kv_head": 6, |
| "head_dim": 64, |
| "causal": true |
| }, |
| "rope": { |
| "theta": 100000.0, |
| "convention": "rotate-half over the two halves of head_dim, applied after QK-Norm" |
| }, |
| "norm": "pre-RMSNorm (eps 1e-05), computed in fp32", |
| "ffn": "SwiGLU: down(silu(gate(x)) * up(x)), d_ff 1920", |
| "bias": false, |
| "residual_init": "output projections (attn.o, mlp.down) std/sqrt(2*n_layer)", |
| "tokenizer": { |
| "name": "cand5-j1tok-24k", |
| "tokenizer_json_sha256": "176a89e2197c24d3f024fbb60ce6f346635e7cba912b3579d93b37d486c9adf1", |
| "vocab_size": 24576, |
| "control_ids": "0-63", |
| "byte_fallback_ids": "64-319", |
| "learned_ids": "320-24575", |
| "eos_token_id": 0, |
| "pad_token_id": 0, |
| "encode_special_tokens": true |
| }, |
| "state_dict_keys": [ |
| "blocks.0.attn.k.weight", |
| "blocks.0.attn.k_norm.weight", |
| "blocks.0.attn.o.weight", |
| "blocks.0.attn.q.weight", |
| "blocks.0.attn.q_norm.weight", |
| "blocks.0.attn.v.weight", |
| "blocks.0.attn_norm.weight", |
| "blocks.0.mlp.down.weight", |
| "blocks.0.mlp.gate.weight", |
| "blocks.0.mlp.up.weight", |
| "blocks.0.mlp_norm.weight", |
| "blocks.1.attn.k.weight", |
| "blocks.1.attn.k_norm.weight", |
| "blocks.1.attn.o.weight", |
| "blocks.1.attn.q.weight", |
| "blocks.1.attn.q_norm.weight", |
| "blocks.1.attn.v.weight", |
| "blocks.1.attn_norm.weight", |
| "blocks.1.mlp.down.weight", |
| "blocks.1.mlp.gate.weight", |
| "blocks.1.mlp.up.weight", |
| "blocks.1.mlp_norm.weight", |
| "blocks.10.attn.k.weight", |
| "blocks.10.attn.k_norm.weight", |
| "blocks.10.attn.o.weight", |
| "blocks.10.attn.q.weight", |
| "blocks.10.attn.q_norm.weight", |
| "blocks.10.attn.v.weight", |
| "blocks.10.attn_norm.weight", |
| "blocks.10.mlp.down.weight", |
| "blocks.10.mlp.gate.weight", |
| "blocks.10.mlp.up.weight", |
| "blocks.10.mlp_norm.weight", |
| "blocks.11.attn.k.weight", |
| "blocks.11.attn.k_norm.weight", |
| "blocks.11.attn.o.weight", |
| "blocks.11.attn.q.weight", |
| "blocks.11.attn.q_norm.weight", |
| "blocks.11.attn.v.weight", |
| "blocks.11.attn_norm.weight", |
| "blocks.11.mlp.down.weight", |
| "blocks.11.mlp.gate.weight", |
| "blocks.11.mlp.up.weight", |
| "blocks.11.mlp_norm.weight", |
| "blocks.12.attn.k.weight", |
| "blocks.12.attn.k_norm.weight", |
| "blocks.12.attn.o.weight", |
| "blocks.12.attn.q.weight", |
| "blocks.12.attn.q_norm.weight", |
| "blocks.12.attn.v.weight", |
| "blocks.12.attn_norm.weight", |
| "blocks.12.mlp.down.weight", |
| "blocks.12.mlp.gate.weight", |
| "blocks.12.mlp.up.weight", |
| "blocks.12.mlp_norm.weight", |
| "blocks.13.attn.k.weight", |
| "blocks.13.attn.k_norm.weight", |
| "blocks.13.attn.o.weight", |
| "blocks.13.attn.q.weight", |
| "blocks.13.attn.q_norm.weight", |
| "blocks.13.attn.v.weight", |
| "blocks.13.attn_norm.weight", |
| "blocks.13.mlp.down.weight", |
| "blocks.13.mlp.gate.weight", |
| "blocks.13.mlp.up.weight", |
| "blocks.13.mlp_norm.weight", |
| "blocks.14.attn.k.weight", |
| "blocks.14.attn.k_norm.weight", |
| "blocks.14.attn.o.weight", |
| "blocks.14.attn.q.weight", |
| "blocks.14.attn.q_norm.weight", |
| "blocks.14.attn.v.weight", |
| "blocks.14.attn_norm.weight", |
| "blocks.14.mlp.down.weight", |
| "blocks.14.mlp.gate.weight", |
| "blocks.14.mlp.up.weight", |
| "blocks.14.mlp_norm.weight", |
| "blocks.15.attn.k.weight", |
| "blocks.15.attn.k_norm.weight", |
| "blocks.15.attn.o.weight", |
| "blocks.15.attn.q.weight", |
| "blocks.15.attn.q_norm.weight", |
| "blocks.15.attn.v.weight", |
| "blocks.15.attn_norm.weight", |
| "blocks.15.mlp.down.weight", |
| "blocks.15.mlp.gate.weight", |
| "blocks.15.mlp.up.weight", |
| "blocks.15.mlp_norm.weight", |
| "blocks.16.attn.k.weight", |
| "blocks.16.attn.k_norm.weight", |
| "blocks.16.attn.o.weight", |
| "blocks.16.attn.q.weight", |
| "blocks.16.attn.q_norm.weight", |
| "blocks.16.attn.v.weight", |
| "blocks.16.attn_norm.weight", |
| "blocks.16.mlp.down.weight", |
| "blocks.16.mlp.gate.weight", |
| "blocks.16.mlp.up.weight", |
| "blocks.16.mlp_norm.weight", |
| "blocks.17.attn.k.weight", |
| "blocks.17.attn.k_norm.weight", |
| "blocks.17.attn.o.weight", |
| "blocks.17.attn.q.weight", |
| "blocks.17.attn.q_norm.weight", |
| "blocks.17.attn.v.weight", |
| "blocks.17.attn_norm.weight", |
| "blocks.17.mlp.down.weight", |
| "blocks.17.mlp.gate.weight", |
| "blocks.17.mlp.up.weight", |
| "blocks.17.mlp_norm.weight", |
| "blocks.18.attn.k.weight", |
| "blocks.18.attn.k_norm.weight", |
| "blocks.18.attn.o.weight", |
| "blocks.18.attn.q.weight", |
| "blocks.18.attn.q_norm.weight", |
| "blocks.18.attn.v.weight", |
| "blocks.18.attn_norm.weight", |
| "blocks.18.mlp.down.weight", |
| "blocks.18.mlp.gate.weight", |
| "blocks.18.mlp.up.weight", |
| "blocks.18.mlp_norm.weight", |
| "blocks.19.attn.k.weight", |
| "blocks.19.attn.k_norm.weight", |
| "blocks.19.attn.o.weight", |
| "blocks.19.attn.q.weight", |
| "blocks.19.attn.q_norm.weight", |
| "blocks.19.attn.v.weight", |
| "blocks.19.attn_norm.weight", |
| "blocks.19.mlp.down.weight", |
| "blocks.19.mlp.gate.weight", |
| "blocks.19.mlp.up.weight", |
| "blocks.19.mlp_norm.weight", |
| "blocks.2.attn.k.weight", |
| "blocks.2.attn.k_norm.weight", |
| "blocks.2.attn.o.weight", |
| "blocks.2.attn.q.weight", |
| "blocks.2.attn.q_norm.weight", |
| "blocks.2.attn.v.weight", |
| "blocks.2.attn_norm.weight", |
| "blocks.2.mlp.down.weight", |
| "blocks.2.mlp.gate.weight", |
| "blocks.2.mlp.up.weight", |
| "blocks.2.mlp_norm.weight", |
| "blocks.3.attn.k.weight", |
| "blocks.3.attn.k_norm.weight", |
| "blocks.3.attn.o.weight", |
| "blocks.3.attn.q.weight", |
| "blocks.3.attn.q_norm.weight", |
| "blocks.3.attn.v.weight", |
| "blocks.3.attn_norm.weight", |
| "blocks.3.mlp.down.weight", |
| "blocks.3.mlp.gate.weight", |
| "blocks.3.mlp.up.weight", |
| "blocks.3.mlp_norm.weight", |
| "blocks.4.attn.k.weight", |
| "blocks.4.attn.k_norm.weight", |
| "blocks.4.attn.o.weight", |
| "blocks.4.attn.q.weight", |
| "blocks.4.attn.q_norm.weight", |
| "blocks.4.attn.v.weight", |
| "blocks.4.attn_norm.weight", |
| "blocks.4.mlp.down.weight", |
| "blocks.4.mlp.gate.weight", |
| "blocks.4.mlp.up.weight", |
| "blocks.4.mlp_norm.weight", |
| "blocks.5.attn.k.weight", |
| "blocks.5.attn.k_norm.weight", |
| "blocks.5.attn.o.weight", |
| "blocks.5.attn.q.weight", |
| "blocks.5.attn.q_norm.weight", |
| "blocks.5.attn.v.weight", |
| "blocks.5.attn_norm.weight", |
| "blocks.5.mlp.down.weight", |
| "blocks.5.mlp.gate.weight", |
| "blocks.5.mlp.up.weight", |
| "blocks.5.mlp_norm.weight", |
| "blocks.6.attn.k.weight", |
| "blocks.6.attn.k_norm.weight", |
| "blocks.6.attn.o.weight", |
| "blocks.6.attn.q.weight", |
| "blocks.6.attn.q_norm.weight", |
| "blocks.6.attn.v.weight", |
| "blocks.6.attn_norm.weight", |
| "blocks.6.mlp.down.weight", |
| "blocks.6.mlp.gate.weight", |
| "blocks.6.mlp.up.weight", |
| "blocks.6.mlp_norm.weight", |
| "blocks.7.attn.k.weight", |
| "blocks.7.attn.k_norm.weight", |
| "blocks.7.attn.o.weight", |
| "blocks.7.attn.q.weight", |
| "blocks.7.attn.q_norm.weight", |
| "blocks.7.attn.v.weight", |
| "blocks.7.attn_norm.weight", |
| "blocks.7.mlp.down.weight", |
| "blocks.7.mlp.gate.weight", |
| "blocks.7.mlp.up.weight", |
| "blocks.7.mlp_norm.weight", |
| "blocks.8.attn.k.weight", |
| "blocks.8.attn.k_norm.weight", |
| "blocks.8.attn.o.weight", |
| "blocks.8.attn.q.weight", |
| "blocks.8.attn.q_norm.weight", |
| "blocks.8.attn.v.weight", |
| "blocks.8.attn_norm.weight", |
| "blocks.8.mlp.down.weight", |
| "blocks.8.mlp.gate.weight", |
| "blocks.8.mlp.up.weight", |
| "blocks.8.mlp_norm.weight", |
| "blocks.9.attn.k.weight", |
| "blocks.9.attn.k_norm.weight", |
| "blocks.9.attn.o.weight", |
| "blocks.9.attn.q.weight", |
| "blocks.9.attn.q_norm.weight", |
| "blocks.9.attn.v.weight", |
| "blocks.9.attn_norm.weight", |
| "blocks.9.mlp.down.weight", |
| "blocks.9.mlp.gate.weight", |
| "blocks.9.mlp.up.weight", |
| "blocks.9.mlp_norm.weight", |
| "embed.weight", |
| "final_norm.weight" |
| ], |
| "tensor_tree_sha256": "1fd07c0d0c6c6b355460ab82ebe02530047b5682a4e6a41b9e4f0146323076e4", |
| "qk_norm_description": "rmsnorm_per_head (learned RMSNorm over head_dim on q and k, before RoPE)" |
| } |
|
|
