{
 "architectures": [
  "LuauCoderForCausalLM"
 ],
 "model_type": "luau_coder",
 "auto_map": {
  "AutoConfig": "configuration_luau_coder.LuauCoderConfig",
  "AutoModel": "modeling_luau_coder.LuauCoderModel",
  "AutoModelForCausalLM": "modeling_luau_coder.LuauCoderForCausalLM"
 },
 "vocab_size": 32768,
 "hidden_size": 1408,
 "intermediate_size": 3840,
 "num_hidden_layers": 28,
 "num_attention_heads": 10,
 "num_key_value_heads": 2,
 "head_dim": 128,
 "rms_norm_eps": 1e-06,
 "final_logit_softcapping": 30.0,
 "max_position_embeddings": 16384,
 "rope_theta": 500000.0,
 "rope_scaling": {
  "rope_type": "yarn",
  "factor": 4.0,
  "original_max_position_embeddings": 4096,
  "beta_fast": 32.0,
  "beta_slow": 1.0,
  "truncate": true
 },
 "nope_layers": [
  3,
  7,
  11,
  15,
  19,
  23,
  27
 ],
 "diff_v2": true,
 "xsa": true,
 "qk_norm": true,
 "use_output_gate": null,
 "attn_res": true,
 "attn_res_block_size": 4,
 "attn_res_delta_block": false,
 "attn_res_partial_source": false,
 "msar": false,
 "ccgqa": true,
 "ccgqa_seq_kernel": 2,
 "ccgqa_head_kernel": 2,
 "ccgqa_qk_mean": true,
 "ccgqa_value_shift": true,
 "vha": true,
 "iha": false,
 "iha_pseudo_heads": 2,
 "iha_window": 0,
 "iha_global_every": 5,
 "initializer_range": 0.026650089544451305,
 "tie_word_embeddings": true,
 "lerope_alpha": null,
 "bos_token_id": null,
 "eos_token_id": 32744,
 "pad_token_id": 32745,
 "torch_dtype": "bfloat16",
 "name": "kai"
}