Uchiha80971 commited on
Commit
46ceb0d
·
verified ·
1 Parent(s): 6627a52

Upload 9 files

Browse files
config.json ADDED
@@ -0,0 +1,18 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "model_type": "sentinel_custom",
3
+ "architectures": [
4
+ "SentinelModel"
5
+ ],
6
+ "vocab_size": 7053,
7
+ "hidden_size": 768,
8
+ "num_layers": 12,
9
+ "num_heads": 12,
10
+ "intermediate_size": 3072,
11
+ "max_position_embeddings": 768,
12
+ "dropout": 0.1,
13
+ "pad_token_id": 0,
14
+ "unk_token_id": 1,
15
+ "bos_token_id": 2,
16
+ "eos_token_id": 3,
17
+ "note": "Custom PyTorch architecture; not directly loadable via AutoModelForCausalLM without custom modeling code."
18
+ }
model/__init__.py ADDED
File without changes
model/__pycache__/__init__.cpython-311.pyc ADDED
Binary file (137 Bytes). View file
 
model/__pycache__/config.cpython-311.pyc ADDED
Binary file (1.08 kB). View file
 
model/__pycache__/sentinel_model.cpython-311.pyc ADDED
Binary file (4.92 kB). View file
 
model/config.py ADDED
@@ -0,0 +1,15 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from dataclasses import dataclass
2
+
3
+ @dataclass
4
+ class SentinelConfig:
5
+ vocab_size: int = 7053
6
+ hidden_size: int = 768
7
+ num_layers: int = 12
8
+ num_heads: int = 12
9
+ intermediate_size: int = 3072
10
+ max_position_embeddings: int = 768
11
+ dropout: float = 0.1
12
+ pad_token_id: int = 0
13
+ unk_token_id: int = 1
14
+ bos_token_id: int = 2
15
+ eos_token_id: int = 3
model/sentinel_model.py ADDED
@@ -0,0 +1,46 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import torch
2
+ import torch.nn as nn
3
+ from model.config import SentinelConfig
4
+
5
+ class SentinelModel(nn.Module):
6
+ def __init__(self, config=None):
7
+ super().__init__()
8
+ self.config = config or SentinelConfig()
9
+ self.token_embedding = nn.Embedding(self.config.vocab_size, self.config.hidden_size, padding_idx=self.config.pad_token_id)
10
+ self.position_embedding = nn.Embedding(self.config.max_position_embeddings, self.config.hidden_size)
11
+ layer = nn.TransformerEncoderLayer(
12
+ d_model=self.config.hidden_size, nhead=self.config.num_heads,
13
+ dim_feedforward=self.config.intermediate_size, dropout=self.config.dropout,
14
+ activation="gelu", batch_first=True, norm_first=True
15
+ )
16
+ self.transformer = nn.TransformerEncoder(layer, num_layers=self.config.num_layers)
17
+ self.norm = nn.LayerNorm(self.config.hidden_size)
18
+ self.lm_head = nn.Linear(self.config.hidden_size, self.config.vocab_size, bias=False)
19
+ self.lm_head.weight = self.token_embedding.weight
20
+ self.apply(self._init_weights)
21
+
22
+ def _init_weights(self, module):
23
+ if isinstance(module, nn.Linear):
24
+ nn.init.normal_(module.weight, mean=0.0, std=0.02)
25
+ if module.bias is not None:
26
+ nn.init.zeros_(module.bias)
27
+ elif isinstance(module, nn.Embedding):
28
+ nn.init.normal_(module.weight, mean=0.0, std=0.02)
29
+
30
+ def forward(self, input_ids, labels=None):
31
+ _, seq_len = input_ids.shape
32
+ if seq_len > self.config.max_position_embeddings:
33
+ raise ValueError(f"Sequence length {seq_len} exceeds context length {self.config.max_position_embeddings}")
34
+ pos = torch.arange(seq_len, device=input_ids.device).unsqueeze(0)
35
+ x = self.token_embedding(input_ids) + self.position_embedding(pos)
36
+ causal = torch.triu(torch.ones(seq_len, seq_len, device=input_ids.device, dtype=torch.bool), diagonal=1)
37
+ padding = input_ids.eq(self.config.pad_token_id)
38
+ x = self.transformer(x, mask=causal, src_key_padding_mask=padding)
39
+ logits = self.lm_head(self.norm(x))
40
+ loss = None
41
+ if labels is not None:
42
+ loss = nn.CrossEntropyLoss(ignore_index=self.config.pad_token_id)(
43
+ logits[:, :-1, :].contiguous().view(-1, self.config.vocab_size),
44
+ labels[:, 1:].contiguous().view(-1)
45
+ )
46
+ return {"logits": logits, "loss": loss}
pytorch_model.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:bc83f6c465ace8c6e042ca6d354870bdea75af130e09f141c2804f7221a8450a
3
+ size 364306995
sentinel_tokenizer.json ADDED
The diff for this file is too large to render. See raw diff