ThingsAI commited on
Commit
a6a57d5
·
verified ·
1 Parent(s): 38b7e42

Upload folder using huggingface_hub

Browse files
Files changed (3) hide show
  1. config.json +6 -2
  2. configuration_dwarf.py +7 -18
  3. modeling_dwarf.py +62 -71
config.json CHANGED
@@ -19,6 +19,10 @@
19
  "head_dim": 64,
20
  "torch_dtype": "float32",
21
  "transformers_version": "4.45.0",
22
- "bos_token_id": 0,
23
- "eos_token_id": 0
 
 
 
 
24
  }
 
19
  "head_dim": 64,
20
  "torch_dtype": "float32",
21
  "transformers_version": "4.45.0",
22
+ "bos_token_id": 1,
23
+ "eos_token_id": 2,
24
+ "num_hidden_layers": 12,
25
+ "hidden_size": 320,
26
+ "num_attention_heads": 5,
27
+ "num_key_value_heads": 1
28
  }
configuration_dwarf.py CHANGED
@@ -1,25 +1,10 @@
1
- """Dwarf model configuration."""
2
  from transformers import PretrainedConfig
3
 
4
-
5
  class DwarfConfig(PretrainedConfig):
6
  model_type = "dwarf"
7
- has_no_defaults_at_init = True
8
-
9
- def __init__(
10
- self,
11
- vocab_size=8202,
12
- d_model=320,
13
- n_layers=12,
14
- n_heads=5,
15
- n_kv_heads=1,
16
- d_ff=864,
17
- max_seq_len=2048,
18
- rope_theta=10000.0,
19
- norm_eps=1e-5,
20
- head_dim=64,
21
- **kwargs,
22
- ):
23
  self.vocab_size = vocab_size
24
  self.d_model = d_model
25
  self.n_layers = n_layers
@@ -30,4 +15,8 @@ class DwarfConfig(PretrainedConfig):
30
  self.rope_theta = rope_theta
31
  self.norm_eps = norm_eps
32
  self.head_dim = head_dim
 
 
 
 
33
  super().__init__(**kwargs)
 
 
1
  from transformers import PretrainedConfig
2
 
 
3
  class DwarfConfig(PretrainedConfig):
4
  model_type = "dwarf"
5
+ def __init__(self, vocab_size=8202, d_model=320, n_layers=12, n_heads=5,
6
+ n_kv_heads=1, d_ff=864, max_seq_len=2048, rope_theta=10000.0,
7
+ norm_eps=1e-5, head_dim=64, **kwargs):
 
 
 
 
 
 
 
 
 
 
 
 
 
8
  self.vocab_size = vocab_size
9
  self.d_model = d_model
10
  self.n_layers = n_layers
 
15
  self.rope_theta = rope_theta
16
  self.norm_eps = norm_eps
17
  self.head_dim = head_dim
18
+ self.num_hidden_layers = n_layers
19
+ self.hidden_size = d_model
20
+ self.num_attention_heads = n_heads
21
+ self.num_key_value_heads = n_kv_heads
22
  super().__init__(**kwargs)
modeling_dwarf.py CHANGED
@@ -1,77 +1,74 @@
1
- """Dwarf-15M: a 15.54M parameter shell/bash specialist language model."""
2
- import math
3
  import torch
4
  import torch.nn as nn
5
  import torch.nn.functional as F
6
  from transformers import PreTrainedModel, GenerationMixin
7
  from .configuration_dwarf import DwarfConfig
8
 
9
-
10
  class RMSNorm(nn.Module):
11
  def __init__(self, dim, eps=1e-5):
12
  super().__init__()
13
- self.weight = nn.Parameter(torch.ones(dim))
14
  self.eps = eps
15
-
16
  def forward(self, x):
17
- return x * torch.rsqrt(x.float().pow(2).mean(-1, keepdim=True) + self.eps).to(x.dtype) * self.weight
18
-
19
 
20
  class RotaryEmbedding(nn.Module):
21
- def __init__(self, dim, max_seq_len=2048, theta=10000.0):
22
  super().__init__()
23
- inv_freq = 1.0 / (theta ** (torch.arange(0, dim, 2).float() / dim))
24
- self.register_buffer("inv_freq", inv_freq, persistent=False)
25
- self._build_cache(max_seq_len)
26
-
27
- def _build_cache(self, seq_len):
28
- t = torch.arange(seq_len, dtype=self.inv_freq.dtype)
29
- freqs = torch.outer(t, self.inv_freq)
30
- self.register_buffer("cos_cache", freqs.cos(), persistent=False)
31
- self.register_buffer("sin_cache", freqs.sin(), persistent=False)
32
-
33
- def forward(self, x, offset=0):
34
- seq_len = x.shape[1]
35
- if offset + seq_len > self.cos_cache.shape[0]:
36
- self._build_cache(offset + seq_len)
37
- cos = self.cos_cache[offset:offset + seq_len]
38
- sin = self.sin_cache[offset:offset + seq_len]
39
- return cos, sin
40
-
41
-
42
- def apply_rope(x, cos, sin):
43
- d = x.shape[-1]
44
- x1, x2 = x[..., :d//2], x[..., d//2:]
45
- cos = cos[:x.shape[-2], :d//2].unsqueeze(0).unsqueeze(0)
46
- sin = sin[:x.shape[-2], :d//2].unsqueeze(0).unsqueeze(0)
47
- return torch.cat([x1 * cos - x2 * sin, x2 * cos + x1 * sin], dim=-1)
48
-
 
 
49
 
50
  class GQAAttention(nn.Module):
51
  def __init__(self, config):
52
  super().__init__()
53
  self.n_heads = config.n_heads
54
  self.n_kv_heads = config.n_kv_heads
 
55
  self.head_dim = config.head_dim
56
- self.q_proj = nn.Linear(config.d_model, config.n_heads * config.head_dim, bias=False)
57
- self.k_proj = nn.Linear(config.d_model, config.n_kv_heads * config.head_dim, bias=False)
58
- self.v_proj = nn.Linear(config.d_model, config.n_kv_heads * config.head_dim, bias=False)
59
  self.o_proj = nn.Linear(config.n_heads * config.head_dim, config.d_model, bias=False)
60
- self.group_size = config.n_heads // config.n_kv_heads
61
-
62
- def forward(self, x, cos, sin):
63
  B, T, _ = x.shape
64
  q = self.q_proj(x).view(B, T, self.n_heads, self.head_dim).transpose(1, 2)
65
  k = self.k_proj(x).view(B, T, self.n_kv_heads, self.head_dim).transpose(1, 2)
66
  v = self.v_proj(x).view(B, T, self.n_kv_heads, self.head_dim).transpose(1, 2)
67
- q = apply_rope(q, cos, sin)
68
- k = apply_rope(k, cos, sin)
69
- if self.group_size > 1:
70
- k = k.unsqueeze(2).expand(-1, -1, self.group_size, -1, -1).reshape(B, self.n_heads, T, self.head_dim)
71
- v = v.unsqueeze(2).expand(-1, -1, self.group_size, -1, -1).reshape(B, self.n_heads, T, self.head_dim)
72
  out = F.scaled_dot_product_attention(q, k, v, is_causal=True)
73
- return self.o_proj(out.transpose(1, 2).reshape(B, T, -1))
74
-
75
 
76
  class SwiGLUFFN(nn.Module):
77
  def __init__(self, config):
@@ -79,58 +76,52 @@ class SwiGLUFFN(nn.Module):
79
  self.gate_proj = nn.Linear(config.d_model, config.d_ff, bias=False)
80
  self.up_proj = nn.Linear(config.d_model, config.d_ff, bias=False)
81
  self.down_proj = nn.Linear(config.d_ff, config.d_model, bias=False)
82
-
83
  def forward(self, x):
84
  return self.down_proj(F.silu(self.gate_proj(x)) * self.up_proj(x))
85
 
86
-
87
  class DwarfBlock(nn.Module):
88
  def __init__(self, config):
89
  super().__init__()
90
- self.attn_norm = RMSNorm(config.d_model, config.norm_eps)
91
  self.attn = GQAAttention(config)
92
- self.ff_norm = RMSNorm(config.d_model, config.norm_eps)
93
  self.ffn = SwiGLUFFN(config)
94
-
95
- def forward(self, x, cos, sin):
96
- x = x + self.attn(self.attn_norm(x), cos, sin)
97
- x = x + self.ffn(self.ff_norm(x))
98
  return x
99
 
100
-
101
  class DwarfForCausalLM(PreTrainedModel, GenerationMixin):
102
  config_class = DwarfConfig
103
-
104
  def __init__(self, config):
105
  super().__init__(config)
106
  self.embed_tokens = nn.Embedding(config.vocab_size, config.d_model)
107
  self.layers = nn.ModuleList([DwarfBlock(config) for _ in range(config.n_layers)])
108
  self.norm = RMSNorm(config.d_model, config.norm_eps)
109
  self.lm_head = nn.Linear(config.d_model, config.vocab_size, bias=False)
110
- self.lm_head.weight = self.embed_tokens.weight # weight tying
111
- self.rope = RotaryEmbedding(config.head_dim, config.max_seq_len, config.rope_theta)
112
  self.post_init()
113
-
 
 
 
 
 
 
 
 
 
114
  def forward(self, input_ids, attention_mask=None, labels=None, **kwargs):
115
  x = self.embed_tokens(input_ids)
116
- cos, sin = self.rope(x)
117
  for layer in self.layers:
118
- x = layer(x, cos, sin)
119
  logits = self.lm_head(self.norm(x))
120
-
121
  loss = None
122
  if labels is not None:
123
  loss = F.cross_entropy(
124
  logits[:, :-1].contiguous().view(-1, logits.size(-1)),
125
- labels[:, 1:].contiguous().view(-1),
126
- ignore_index=-100,
127
- )
128
-
129
  from transformers.modeling_outputs import CausalLMOutput
130
  return CausalLMOutput(loss=loss, logits=logits)
131
-
132
  def prepare_inputs_for_generation(self, input_ids, **kwargs):
133
  return {"input_ids": input_ids}
134
-
135
- def count_parameters(self):
136
- return sum(p.numel() for p in self.parameters() if p.requires_grad)
 
 
 
1
  import torch
2
  import torch.nn as nn
3
  import torch.nn.functional as F
4
  from transformers import PreTrainedModel, GenerationMixin
5
  from .configuration_dwarf import DwarfConfig
6
 
 
7
  class RMSNorm(nn.Module):
8
  def __init__(self, dim, eps=1e-5):
9
  super().__init__()
 
10
  self.eps = eps
11
+ self.scale = nn.Parameter(torch.ones(dim))
12
  def forward(self, x):
13
+ rms = x.float().pow(2).mean(-1, keepdim=True).add(self.eps).rsqrt()
14
+ return (x.float() * rms).to(x.dtype) * self.scale
15
 
16
  class RotaryEmbedding(nn.Module):
17
+ def __init__(self, head_dim, max_seq_len, theta=10000.0):
18
  super().__init__()
19
+ assert head_dim % 2 == 0
20
+ self.head_dim = head_dim
21
+ self.max_seq_len = max_seq_len
22
+ self.theta = theta
23
+ self.cos_cache = None
24
+ self.sin_cache = None
25
+ self._max = 0
26
+ def _build_cache(self, seq_len, device):
27
+ inv_freq = 1.0 / (self.theta ** (torch.arange(0, self.head_dim, 2, device=device).float() / self.head_dim))
28
+ t = torch.arange(seq_len, device=device).float()
29
+ freqs = torch.outer(t, inv_freq)
30
+ emb = torch.cat([freqs, freqs], dim=-1)
31
+ self.cos_cache = emb.cos()[None, None]
32
+ self.sin_cache = emb.sin()[None, None]
33
+ self._max = seq_len
34
+ @staticmethod
35
+ def _rotate_half(x):
36
+ x1, x2 = x.chunk(2, dim=-1)
37
+ return torch.cat([-x2, x1], dim=-1)
38
+ def forward(self, q, k):
39
+ T = q.size(2)
40
+ if self.cos_cache is None or T > self._max or self.cos_cache.device != q.device:
41
+ self._build_cache(max(T, self.max_seq_len), q.device)
42
+ cos = self.cos_cache[:, :, :T, :]
43
+ sin = self.sin_cache[:, :, :T, :]
44
+ q = q * cos + self._rotate_half(q) * sin
45
+ k = k * cos + self._rotate_half(k) * sin
46
+ return q, k
47
 
48
  class GQAAttention(nn.Module):
49
  def __init__(self, config):
50
  super().__init__()
51
  self.n_heads = config.n_heads
52
  self.n_kv_heads = config.n_kv_heads
53
+ self.n_groups = config.n_heads // config.n_kv_heads
54
  self.head_dim = config.head_dim
55
+ self.q_proj = nn.Linear(config.d_model, config.n_heads * config.head_dim, bias=True)
56
+ self.k_proj = nn.Linear(config.d_model, config.n_kv_heads * config.head_dim, bias=True)
57
+ self.v_proj = nn.Linear(config.d_model, config.n_kv_heads * config.head_dim, bias=True)
58
  self.o_proj = nn.Linear(config.n_heads * config.head_dim, config.d_model, bias=False)
59
+ self.rope = RotaryEmbedding(config.head_dim, config.max_seq_len, config.rope_theta)
60
+ def forward(self, x):
 
61
  B, T, _ = x.shape
62
  q = self.q_proj(x).view(B, T, self.n_heads, self.head_dim).transpose(1, 2)
63
  k = self.k_proj(x).view(B, T, self.n_kv_heads, self.head_dim).transpose(1, 2)
64
  v = self.v_proj(x).view(B, T, self.n_kv_heads, self.head_dim).transpose(1, 2)
65
+ q, k = self.rope(q, k)
66
+ if self.n_groups > 1:
67
+ k = k.repeat_interleave(self.n_groups, dim=1)
68
+ v = v.repeat_interleave(self.n_groups, dim=1)
 
69
  out = F.scaled_dot_product_attention(q, k, v, is_causal=True)
70
+ out = out.transpose(1, 2).contiguous().view(B, T, self.n_heads * self.head_dim)
71
+ return self.o_proj(out)
72
 
73
  class SwiGLUFFN(nn.Module):
74
  def __init__(self, config):
 
76
  self.gate_proj = nn.Linear(config.d_model, config.d_ff, bias=False)
77
  self.up_proj = nn.Linear(config.d_model, config.d_ff, bias=False)
78
  self.down_proj = nn.Linear(config.d_ff, config.d_model, bias=False)
 
79
  def forward(self, x):
80
  return self.down_proj(F.silu(self.gate_proj(x)) * self.up_proj(x))
81
 
 
82
  class DwarfBlock(nn.Module):
83
  def __init__(self, config):
84
  super().__init__()
85
+ self.norm_attn = RMSNorm(config.d_model, config.norm_eps)
86
  self.attn = GQAAttention(config)
87
+ self.norm_ffn = RMSNorm(config.d_model, config.norm_eps)
88
  self.ffn = SwiGLUFFN(config)
89
+ def forward(self, x):
90
+ x = x + self.attn(self.norm_attn(x))
91
+ x = x + self.ffn(self.norm_ffn(x))
 
92
  return x
93
 
 
94
  class DwarfForCausalLM(PreTrainedModel, GenerationMixin):
95
  config_class = DwarfConfig
96
+ _tied_weights_keys = ["lm_head.weight"]
97
  def __init__(self, config):
98
  super().__init__(config)
99
  self.embed_tokens = nn.Embedding(config.vocab_size, config.d_model)
100
  self.layers = nn.ModuleList([DwarfBlock(config) for _ in range(config.n_layers)])
101
  self.norm = RMSNorm(config.d_model, config.norm_eps)
102
  self.lm_head = nn.Linear(config.d_model, config.vocab_size, bias=False)
 
 
103
  self.post_init()
104
+ def tie_weights(self, **kwargs):
105
+ self.lm_head.weight = self.embed_tokens.weight
106
+ def get_input_embeddings(self):
107
+ return self.embed_tokens
108
+ def set_input_embeddings(self, value):
109
+ self.embed_tokens = value
110
+ def get_output_embeddings(self):
111
+ return self.lm_head
112
+ def set_output_embeddings(self, new_embeddings):
113
+ self.lm_head = new_embeddings
114
  def forward(self, input_ids, attention_mask=None, labels=None, **kwargs):
115
  x = self.embed_tokens(input_ids)
 
116
  for layer in self.layers:
117
+ x = layer(x)
118
  logits = self.lm_head(self.norm(x))
 
119
  loss = None
120
  if labels is not None:
121
  loss = F.cross_entropy(
122
  logits[:, :-1].contiguous().view(-1, logits.size(-1)),
123
+ labels[:, 1:].contiguous().view(-1), ignore_index=-100)
 
 
 
124
  from transformers.modeling_outputs import CausalLMOutput
125
  return CausalLMOutput(loss=loss, logits=logits)
 
126
  def prepare_inputs_for_generation(self, input_ids, **kwargs):
127
  return {"input_ids": input_ids}