User01110 commited on
Commit
48da242
·
verified ·
1 Parent(s): 55856f7

Embed custom config in remote code

Browse files
Files changed (1) hide show
  1. modeling_tinyqwen3_novelty.py +43 -2
modeling_tinyqwen3_novelty.py CHANGED
@@ -1,10 +1,51 @@
 
1
  import torch
2
  import torch.nn as nn
3
  import torch.nn.functional as F
4
  from transformers import PreTrainedModel
 
5
  from transformers.generation import GenerationMixin
6
  from transformers.modeling_outputs import CausalLMOutput
7
- from transformers.models.qwen3.configuration_qwen3 import Qwen3Config
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
8
 
9
 
10
  class RMSNorm(nn.Module):
@@ -123,7 +164,7 @@ class TinyQwen3NoveltyBlock(nn.Module):
123
 
124
 
125
  class TinyQwen3NoveltyForCausalLM(PreTrainedModel, GenerationMixin):
126
- config_class = Qwen3Config
127
  base_model_prefix = ""
128
  _no_split_modules = ["TinyQwen3NoveltyBlock"]
129
  _tied_weights_keys = {}
 
1
+
2
  import torch
3
  import torch.nn as nn
4
  import torch.nn.functional as F
5
  from transformers import PreTrainedModel
6
+ from transformers.configuration_utils import PretrainedConfig
7
  from transformers.generation import GenerationMixin
8
  from transformers.modeling_outputs import CausalLMOutput
9
+
10
+
11
+ class TinyQwen3NoveltyConfig(PretrainedConfig):
12
+ model_type = "tinyqwen3_novelty"
13
+
14
+ def __init__(
15
+ self,
16
+ vocab_size=4098,
17
+ hidden_size=256,
18
+ intermediate_size=896,
19
+ num_hidden_layers=8,
20
+ num_attention_heads=8,
21
+ num_key_value_heads=4,
22
+ head_dim=32,
23
+ rms_norm_eps=1e-6,
24
+ rope_theta=2500.0,
25
+ max_position_embeddings=512,
26
+ tie_word_embeddings=True,
27
+ novelty_gate_floor=0.05,
28
+ novelty_gate_type="math_rms_abs_delta",
29
+ im_start_token_id=4096,
30
+ im_end_token_id=4097,
31
+ **kwargs,
32
+ ):
33
+ super().__init__(tie_word_embeddings=tie_word_embeddings, **kwargs)
34
+ self.vocab_size = vocab_size
35
+ self.hidden_size = hidden_size
36
+ self.intermediate_size = intermediate_size
37
+ self.num_hidden_layers = num_hidden_layers
38
+ self.num_attention_heads = num_attention_heads
39
+ self.num_key_value_heads = num_key_value_heads
40
+ self.head_dim = head_dim
41
+ self.rms_norm_eps = rms_norm_eps
42
+ self.rope_theta = rope_theta
43
+ self.max_position_embeddings = max_position_embeddings
44
+ self.tie_word_embeddings = tie_word_embeddings
45
+ self.novelty_gate_floor = novelty_gate_floor
46
+ self.novelty_gate_type = novelty_gate_type
47
+ self.im_start_token_id = im_start_token_id
48
+ self.im_end_token_id = im_end_token_id
49
 
50
 
51
  class RMSNorm(nn.Module):
 
164
 
165
 
166
  class TinyQwen3NoveltyForCausalLM(PreTrainedModel, GenerationMixin):
167
+ config_class = TinyQwen3NoveltyConfig
168
  base_model_prefix = ""
169
  _no_split_modules = ["TinyQwen3NoveltyBlock"]
170
  _tied_weights_keys = {}