Upload 9 files
Browse files- config.json +18 -0
- model/__init__.py +0 -0
- model/__pycache__/__init__.cpython-311.pyc +0 -0
- model/__pycache__/config.cpython-311.pyc +0 -0
- model/__pycache__/sentinel_model.cpython-311.pyc +0 -0
- model/config.py +15 -0
- model/sentinel_model.py +46 -0
- pytorch_model.bin +3 -0
- sentinel_tokenizer.json +0 -0
config.json
ADDED
|
@@ -0,0 +1,18 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"model_type": "sentinel_custom",
|
| 3 |
+
"architectures": [
|
| 4 |
+
"SentinelModel"
|
| 5 |
+
],
|
| 6 |
+
"vocab_size": 7053,
|
| 7 |
+
"hidden_size": 768,
|
| 8 |
+
"num_layers": 12,
|
| 9 |
+
"num_heads": 12,
|
| 10 |
+
"intermediate_size": 3072,
|
| 11 |
+
"max_position_embeddings": 768,
|
| 12 |
+
"dropout": 0.1,
|
| 13 |
+
"pad_token_id": 0,
|
| 14 |
+
"unk_token_id": 1,
|
| 15 |
+
"bos_token_id": 2,
|
| 16 |
+
"eos_token_id": 3,
|
| 17 |
+
"note": "Custom PyTorch architecture; not directly loadable via AutoModelForCausalLM without custom modeling code."
|
| 18 |
+
}
|
model/__init__.py
ADDED
|
File without changes
|
model/__pycache__/__init__.cpython-311.pyc
ADDED
|
Binary file (137 Bytes). View file
|
|
|
model/__pycache__/config.cpython-311.pyc
ADDED
|
Binary file (1.08 kB). View file
|
|
|
model/__pycache__/sentinel_model.cpython-311.pyc
ADDED
|
Binary file (4.92 kB). View file
|
|
|
model/config.py
ADDED
|
@@ -0,0 +1,15 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from dataclasses import dataclass
|
| 2 |
+
|
| 3 |
+
@dataclass
|
| 4 |
+
class SentinelConfig:
|
| 5 |
+
vocab_size: int = 7053
|
| 6 |
+
hidden_size: int = 768
|
| 7 |
+
num_layers: int = 12
|
| 8 |
+
num_heads: int = 12
|
| 9 |
+
intermediate_size: int = 3072
|
| 10 |
+
max_position_embeddings: int = 768
|
| 11 |
+
dropout: float = 0.1
|
| 12 |
+
pad_token_id: int = 0
|
| 13 |
+
unk_token_id: int = 1
|
| 14 |
+
bos_token_id: int = 2
|
| 15 |
+
eos_token_id: int = 3
|
model/sentinel_model.py
ADDED
|
@@ -0,0 +1,46 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import torch
|
| 2 |
+
import torch.nn as nn
|
| 3 |
+
from model.config import SentinelConfig
|
| 4 |
+
|
| 5 |
+
class SentinelModel(nn.Module):
|
| 6 |
+
def __init__(self, config=None):
|
| 7 |
+
super().__init__()
|
| 8 |
+
self.config = config or SentinelConfig()
|
| 9 |
+
self.token_embedding = nn.Embedding(self.config.vocab_size, self.config.hidden_size, padding_idx=self.config.pad_token_id)
|
| 10 |
+
self.position_embedding = nn.Embedding(self.config.max_position_embeddings, self.config.hidden_size)
|
| 11 |
+
layer = nn.TransformerEncoderLayer(
|
| 12 |
+
d_model=self.config.hidden_size, nhead=self.config.num_heads,
|
| 13 |
+
dim_feedforward=self.config.intermediate_size, dropout=self.config.dropout,
|
| 14 |
+
activation="gelu", batch_first=True, norm_first=True
|
| 15 |
+
)
|
| 16 |
+
self.transformer = nn.TransformerEncoder(layer, num_layers=self.config.num_layers)
|
| 17 |
+
self.norm = nn.LayerNorm(self.config.hidden_size)
|
| 18 |
+
self.lm_head = nn.Linear(self.config.hidden_size, self.config.vocab_size, bias=False)
|
| 19 |
+
self.lm_head.weight = self.token_embedding.weight
|
| 20 |
+
self.apply(self._init_weights)
|
| 21 |
+
|
| 22 |
+
def _init_weights(self, module):
|
| 23 |
+
if isinstance(module, nn.Linear):
|
| 24 |
+
nn.init.normal_(module.weight, mean=0.0, std=0.02)
|
| 25 |
+
if module.bias is not None:
|
| 26 |
+
nn.init.zeros_(module.bias)
|
| 27 |
+
elif isinstance(module, nn.Embedding):
|
| 28 |
+
nn.init.normal_(module.weight, mean=0.0, std=0.02)
|
| 29 |
+
|
| 30 |
+
def forward(self, input_ids, labels=None):
|
| 31 |
+
_, seq_len = input_ids.shape
|
| 32 |
+
if seq_len > self.config.max_position_embeddings:
|
| 33 |
+
raise ValueError(f"Sequence length {seq_len} exceeds context length {self.config.max_position_embeddings}")
|
| 34 |
+
pos = torch.arange(seq_len, device=input_ids.device).unsqueeze(0)
|
| 35 |
+
x = self.token_embedding(input_ids) + self.position_embedding(pos)
|
| 36 |
+
causal = torch.triu(torch.ones(seq_len, seq_len, device=input_ids.device, dtype=torch.bool), diagonal=1)
|
| 37 |
+
padding = input_ids.eq(self.config.pad_token_id)
|
| 38 |
+
x = self.transformer(x, mask=causal, src_key_padding_mask=padding)
|
| 39 |
+
logits = self.lm_head(self.norm(x))
|
| 40 |
+
loss = None
|
| 41 |
+
if labels is not None:
|
| 42 |
+
loss = nn.CrossEntropyLoss(ignore_index=self.config.pad_token_id)(
|
| 43 |
+
logits[:, :-1, :].contiguous().view(-1, self.config.vocab_size),
|
| 44 |
+
labels[:, 1:].contiguous().view(-1)
|
| 45 |
+
)
|
| 46 |
+
return {"logits": logits, "loss": loss}
|
pytorch_model.bin
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:bc83f6c465ace8c6e042ca6d354870bdea75af130e09f141c2804f7221a8450a
|
| 3 |
+
size 364306995
|
sentinel_tokenizer.json
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|