bebrws commited on
Commit
c075c01
·
verified ·
1 Parent(s): b7a81c4

Fix RoPE config: restore rope_theta=1e6, drop static YaRN, native 32k context

Browse files
Files changed (1) hide show
  1. config.json +3 -7
config.json CHANGED
@@ -50,7 +50,7 @@
50
  "full_attention",
51
  "full_attention"
52
  ],
53
- "max_position_embeddings": 131072,
54
  "max_window_layers": 36,
55
  "model_type": "qwen3",
56
  "num_attention_heads": 32,
@@ -59,7 +59,7 @@
59
  "pad_token_id": 151643,
60
  "rms_norm_eps": 1e-06,
61
  "rope_parameters": {
62
- "rope_theta": 1000000,
63
  "rope_type": "default"
64
  },
65
  "sliding_window": null,
@@ -68,9 +68,5 @@
68
  "use_cache": false,
69
  "use_sliding_window": false,
70
  "vocab_size": 151936,
71
- "rope_scaling": {
72
- "rope_type": "yarn",
73
- "factor": 4.0,
74
- "original_max_position_embeddings": 32768
75
- }
76
  }
 
50
  "full_attention",
51
  "full_attention"
52
  ],
53
+ "max_position_embeddings": 32768,
54
  "max_window_layers": 36,
55
  "model_type": "qwen3",
56
  "num_attention_heads": 32,
 
59
  "pad_token_id": 151643,
60
  "rms_norm_eps": 1e-06,
61
  "rope_parameters": {
62
+ "rope_theta": 1000000.0,
63
  "rope_type": "default"
64
  },
65
  "sliding_window": null,
 
68
  "use_cache": false,
69
  "use_sliding_window": false,
70
  "vocab_size": 151936,
71
+ "rope_theta": 1000000.0
 
 
 
 
72
  }