{ "architectures": [ "SonkeForCausalLM" ], "model_type": "sonke", "auto_map": { "AutoConfig": "modeling_sonke.SonkeConfig", "AutoModelForCausalLM": "modeling_sonke.SonkeForCausalLM" }, "vocab_size": 32768, "hidden_size": 1024, "num_hidden_layers": 26, "num_attention_heads": 16, "num_key_value_heads": 4, "intermediate_size": 2816, "rope_theta": 100000.0, "final_logit_softcapping": 30.0, "ngram_rows": 65536, "ngram_dim": 160, "ngram_layer": 1, "rms_norm_eps": 1e-06, "max_position_embeddings": 4096, "bos_token_id": null, "eos_token_id": 3, "pad_token_id": 1, "tie_word_embeddings": true, "torch_dtype": "float32", "training_step": 58666, "training_tokens": 22757507072 }