| { |
| "architectures": [ |
| "LuauCoderForCausalLM" |
| ], |
| "model_type": "luau_coder", |
| "auto_map": { |
| "AutoConfig": "configuration_luau_coder.LuauCoderConfig", |
| "AutoModel": "modeling_luau_coder.LuauCoderModel", |
| "AutoModelForCausalLM": "modeling_luau_coder.LuauCoderForCausalLM" |
| }, |
| "vocab_size": 32768, |
| "hidden_size": 1408, |
| "intermediate_size": 3840, |
| "num_hidden_layers": 28, |
| "num_attention_heads": 10, |
| "num_key_value_heads": 2, |
| "head_dim": 128, |
| "rms_norm_eps": 1e-06, |
| "final_logit_softcapping": 30.0, |
| "max_position_embeddings": 16384, |
| "rope_theta": 500000.0, |
| "rope_scaling": { |
| "rope_type": "yarn", |
| "factor": 4.0, |
| "original_max_position_embeddings": 4096, |
| "beta_fast": 32.0, |
| "beta_slow": 1.0, |
| "truncate": true |
| }, |
| "nope_layers": [ |
| 3, |
| 7, |
| 11, |
| 15, |
| 19, |
| 23, |
| 27 |
| ], |
| "diff_v2": true, |
| "xsa": true, |
| "qk_norm": true, |
| "use_output_gate": null, |
| "attn_res": true, |
| "attn_res_block_size": 4, |
| "attn_res_delta_block": false, |
| "attn_res_partial_source": false, |
| "msar": false, |
| "ccgqa": true, |
| "ccgqa_seq_kernel": 2, |
| "ccgqa_head_kernel": 2, |
| "ccgqa_qk_mean": true, |
| "ccgqa_value_shift": true, |
| "vha": true, |
| "iha": false, |
| "iha_pseudo_heads": 2, |
| "iha_window": 0, |
| "iha_global_every": 5, |
| "initializer_range": 0.026650089544451305, |
| "tie_word_embeddings": true, |
| "lerope_alpha": null, |
| "bos_token_id": null, |
| "eos_token_id": 32744, |
| "pad_token_id": 32745, |
| "torch_dtype": "bfloat16", |
| "name": "kai" |
| } |
