shanjiaz's picture
Publish final DFlash2 config and training validation metadata
fa496ec verified
Raw History Blame Contribute Delete
1.97 kB
{
"architectures": [
"DFlash2DraftModel"
],
"auto_map": {
"": "config.DFlash2SpeculatorConfig"
},
"aux_hidden_state_layer_ids": [
18,
30,
42,
54,
66,
74
],
"block_size": 9,
"conv_group_size": 16,
"conv_kernel_size": 2,
"draft_vocab_size": 154880,
"dtype": "bfloat16",
"mask_token_id": 154856,
"sample_from_anchor": false,
"selector_rank": 256,
"selector_top_k": 16,
"sliding_window_non_causal": true,
"speculators_config": {
"algorithm": "dflash2",
"default_proposal_method": "greedy",
"proposal_methods": [
{
"accept_tolerance": 0.0,
"proposal_type": "greedy",
"speculative_tokens": 8,
"verifier_accept_k": 1
}
],
"verifier": {
"architectures": [
"GlmMoeDsaForCausalLM"
],
"name_or_path": "zai-org/GLM-5.3"
}
},
"speculators_model_type": "dflash2",
"speculators_version": "0.9.0a95",
"target_hidden_size": null,
"tie_word_embeddings": false,
"transformer_layer_config": {
"attention_bias": false,
"attention_dropout": 0.0,
"bos_token_id": null,
"eos_token_id": null,
"head_dim": 64,
"hidden_act": "silu",
"hidden_size": 6144,
"initializer_range": 0.02,
"intermediate_size": 12288,
"layer_types": [
"sliding_attention",
"sliding_attention",
"sliding_attention",
"sliding_attention",
"sliding_attention"
],
"max_position_embeddings": 1048576,
"max_window_layers": 28,
"model_type": "qwen3",
"num_attention_heads": 64,
"num_hidden_layers": 5,
"num_key_value_heads": 64,
"pad_token_id": null,
"rms_norm_eps": 1e-05,
"rope_parameters": {
"rope_theta": 8000000,
"rope_type": "default"
},
"sliding_window": 2048,
"tie_word_embeddings": false,
"use_cache": true,
"use_sliding_window": true,
"vocab_size": 154880
},
"transformers_version": "5.19.0"
}