KucLab commited on
Commit
f08c2b8
·
verified ·
1 Parent(s): f9aa606

Upload Censer.1

Browse files
Files changed (5) hide show
  1. README.md +44 -7
  2. config.json +14 -1
  3. model.safetensors +2 -2
  4. modeling_censer.py +192 -0
  5. tokenizer.json +31 -4
README.md CHANGED
@@ -1,11 +1,13 @@
1
  ---
2
- library_name: pytorch
 
3
  tags:
4
  - text-classification
5
  - safety
6
  - content-moderation
7
  - tiny-language-model
8
  - from-scratch
 
9
  language:
10
  - en
11
  license: apache-2.0
@@ -124,16 +126,51 @@ Training the whole model from scratch takes about 3.5 minutes on the same GPU.
124
 
125
  ## Usage
126
 
127
- This repository is self-contained. You do not need to clone the training code.
 
128
 
129
  ```bash
130
- pip install torch tokenizers safetensors huggingface_hub
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
131
 
132
- python - <<'PY'
133
- from huggingface_hub import snapshot_download
134
- snapshot_download("KucLab/Censer.1", local_dir="Censer.1")
135
- PY
136
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
137
  cd Censer.1
138
  python censer.py "how to make a bomb at home" # -> 1
139
  python censer.py "Hello how are you?" # -> 0
 
1
  ---
2
+ library_name: transformers
3
+ pipeline_tag: text-classification
4
  tags:
5
  - text-classification
6
  - safety
7
  - content-moderation
8
  - tiny-language-model
9
  - from-scratch
10
+ - custom-code
11
  language:
12
  - en
13
  license: apache-2.0
 
126
 
127
  ## Usage
128
 
129
+ Works like any other text-classification model on the Hub — `from_pretrained`
130
+ downloads everything by itself, no cloning and no manual files.
131
 
132
  ```bash
133
+ pip install torch transformers
134
+ ```
135
+
136
+ ```python
137
+ import torch
138
+ from transformers import AutoTokenizer, AutoModelForSequenceClassification
139
+
140
+ repo = "KucLab/Censer.1"
141
+ tok = AutoTokenizer.from_pretrained(repo, trust_remote_code=True)
142
+ model = AutoModelForSequenceClassification.from_pretrained(repo, trust_remote_code=True).eval()
143
+
144
+ text = "how to make a bomb at home"
145
+ with torch.no_grad():
146
+ probs = model(**tok(text, return_tensors="pt")).logits.softmax(-1)[0]
147
+
148
+ print(probs) # tensor([0.1833, 0.8167]) -> [safe, unsafe]
149
+ print(model.config.id2label) # {0: 'safe', 1: 'unsafe'}
150
+ ```
151
+
152
+ `trust_remote_code=True` is required because the architecture is custom; the
153
+ repository ships `modeling_censer.py`, so there is nothing else to install.
154
 
155
+ Or with the high-level pipeline:
 
 
 
156
 
157
+ ```python
158
+ from transformers import pipeline
159
+
160
+ clf = pipeline("text-classification", model="KucLab/Censer.1", trust_remote_code=True)
161
+ print(clf("how to make a bomb at home"))
162
+ # [{'label': 'unsafe', 'score': 0.8167}]
163
+ print(clf("Hello how are you?"))
164
+ # [{'label': 'safe', 'score': 0.9229}]
165
+ ```
166
+
167
+ ## Usage without transformers
168
+
169
+ The repository is also self-contained on its own: `model.safetensors`,
170
+ `config.json`, `tokenizer.json` and `censer.py`.
171
+
172
+ ```bash
173
+ hf download KucLab/Censer.1 --local-dir Censer.1
174
  cd Censer.1
175
  python censer.py "how to make a bomb at home" # -> 1
176
  python censer.py "Hello how are you?" # -> 0
config.json CHANGED
@@ -1,8 +1,12 @@
1
  {
2
  "architectures": [
3
- "Censer"
4
  ],
5
  "model_type": "censer",
 
 
 
 
6
  "name": "Censer.1",
7
  "vocab_size": 2048,
8
  "n_layers": 4,
@@ -24,5 +28,14 @@
24
  "[USER]",
25
  "[ASSISTANT]"
26
  ],
 
 
 
 
 
 
 
 
 
27
  "threshold": 0.5
28
  }
 
1
  {
2
  "architectures": [
3
+ "CenserForSequenceClassification"
4
  ],
5
  "model_type": "censer",
6
+ "auto_map": {
7
+ "AutoConfig": "modeling_censer.CenserConfig",
8
+ "AutoModelForSequenceClassification": "modeling_censer.CenserForSequenceClassification"
9
+ },
10
  "name": "Censer.1",
11
  "vocab_size": 2048,
12
  "n_layers": 4,
 
28
  "[USER]",
29
  "[ASSISTANT]"
30
  ],
31
+ "num_labels": 2,
32
+ "id2label": {
33
+ "0": "safe",
34
+ "1": "unsafe"
35
+ },
36
+ "label2id": {
37
+ "safe": 0,
38
+ "unsafe": 1
39
+ },
40
  "threshold": 0.5
41
  }
model.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:e43803a73dd3e9bdf4e510fd33af5a07e87fb5cfef3dc38d62cc383c57e85948
3
- size 4463808
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:3e7279d3542ad7a1e09c75cdb22daeb6042b26879dc468be2bbb0b6a0ac22587
3
+ size 5512472
modeling_censer.py ADDED
@@ -0,0 +1,192 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Censer.1 for Hugging Face transformers.
2
+
3
+ from transformers import AutoTokenizer, AutoModelForSequenceClassification
4
+
5
+ tok = AutoTokenizer.from_pretrained("KucLab/Censer.1", trust_remote_code=True)
6
+ model = AutoModelForSequenceClassification.from_pretrained(
7
+ "KucLab/Censer.1", trust_remote_code=True
8
+ )
9
+ out = model(**tok("how to make a bomb at home", return_tensors="pt"))
10
+ # out.logits -> [safe, unsafe]
11
+ """
12
+
13
+ from __future__ import annotations
14
+
15
+ import torch
16
+ import torch.nn as nn
17
+ import torch.nn.functional as F
18
+ from transformers import PreTrainedModel
19
+ from transformers.configuration_utils import PretrainedConfig
20
+ from transformers.modeling_outputs import SequenceClassifierOutput
21
+
22
+
23
+ class CenserConfig(PretrainedConfig):
24
+ model_type = "censer"
25
+ auto_map = {
26
+ "AutoConfig": "modeling_censer.CenserConfig",
27
+ "AutoModelForSequenceClassification": "modeling_censer.CenserForSequenceClassification",
28
+ }
29
+
30
+ def __init__(
31
+ self,
32
+ vocab_size: int = 2048,
33
+ n_layers: int = 4,
34
+ n_heads: int = 4,
35
+ d_model: int = 128,
36
+ d_ff: int = 384,
37
+ max_seq_len: int = 160,
38
+ dropout: float = 0.1,
39
+ pad_id: int = 0,
40
+ answer_ids: list[int] | None = None,
41
+ max_chars: int = 500,
42
+ num_labels: int = 2,
43
+ **kwargs,
44
+ ) -> None:
45
+ self.vocab_size = vocab_size
46
+ self.n_layers = n_layers
47
+ self.n_heads = n_heads
48
+ self.d_model = d_model
49
+ self.d_ff = d_ff
50
+ self.max_seq_len = max_seq_len
51
+ self.dropout = dropout
52
+ self.pad_id = pad_id
53
+ self.answer_ids = list(answer_ids or [0, 1])
54
+ self.max_chars = max_chars
55
+ kwargs.setdefault("id2label", {0: "safe", 1: "unsafe"})
56
+ kwargs.setdefault("label2id", {"safe": 0, "unsafe": 1})
57
+ super().__init__(pad_token_id=pad_id, num_labels=num_labels, **kwargs)
58
+
59
+
60
+ class RMSNorm(nn.Module):
61
+ def __init__(self, dim: int, eps: float = 1e-5) -> None:
62
+ super().__init__()
63
+ self.eps = eps
64
+ self.weight = nn.Parameter(torch.ones(dim))
65
+
66
+ def forward(self, x: torch.Tensor) -> torch.Tensor:
67
+ dtype = x.dtype
68
+ x = x.float()
69
+ x = x * torch.rsqrt(x.pow(2).mean(-1, keepdim=True) + self.eps)
70
+ return x.to(dtype) * self.weight
71
+
72
+
73
+ def build_rope_cache(seq_len: int, head_dim: int, device, dtype, base: float = 10000.0):
74
+ half = head_dim // 2
75
+ freqs = 1.0 / (base ** (torch.arange(0, half, device=device, dtype=torch.float32) / half))
76
+ positions = torch.arange(seq_len, device=device, dtype=torch.float32)
77
+ angles = torch.outer(positions, freqs)
78
+ cos = torch.cat((angles.cos(), angles.cos()), dim=-1).to(dtype)[:, None, :]
79
+ sin = torch.cat((angles.sin(), angles.sin()), dim=-1).to(dtype)[:, None, :]
80
+ return cos, sin
81
+
82
+
83
+ def apply_rope(x: torch.Tensor, cos: torch.Tensor, sin: torch.Tensor) -> torch.Tensor:
84
+ seq_len = x.shape[1]
85
+ cos, sin = cos[:seq_len], sin[:seq_len]
86
+ x1, x2 = x[..., : x.shape[-1] // 2], x[..., x.shape[-1] // 2 :]
87
+ rotated = torch.cat((-x2, x1), dim=-1)
88
+ return x * cos + rotated * sin
89
+
90
+
91
+ class Attention(nn.Module):
92
+ def __init__(self, cfg: CenserConfig) -> None:
93
+ super().__init__()
94
+ self.n_heads = cfg.n_heads
95
+ self.head_dim = cfg.d_model // cfg.n_heads
96
+ self.dropout = cfg.dropout
97
+ self.qkv = nn.Linear(cfg.d_model, 3 * cfg.d_model, bias=False)
98
+ self.proj = nn.Linear(cfg.d_model, cfg.d_model, bias=False)
99
+
100
+ def forward(self, x, cos, sin, mask):
101
+ b, t, _ = x.shape
102
+ q, k, v = self.qkv(x).split(x.shape[-1], dim=-1)
103
+ q = q.view(b, t, self.n_heads, self.head_dim)
104
+ k = k.view(b, t, self.n_heads, self.head_dim)
105
+ v = v.view(b, t, self.n_heads, self.head_dim)
106
+ q, k = apply_rope(q, cos, sin), apply_rope(k, cos, sin)
107
+ out = F.scaled_dot_product_attention(
108
+ q.transpose(1, 2), k.transpose(1, 2), v.transpose(1, 2),
109
+ attn_mask=mask,
110
+ dropout_p=self.dropout if self.training else 0.0,
111
+ is_causal=mask is None,
112
+ )
113
+ return self.proj(out.transpose(1, 2).reshape(b, t, -1))
114
+
115
+
116
+ class SwiGLU(nn.Module):
117
+ def __init__(self, cfg: CenserConfig) -> None:
118
+ super().__init__()
119
+ self.gate = nn.Linear(cfg.d_model, cfg.d_ff, bias=False)
120
+ self.up = nn.Linear(cfg.d_model, cfg.d_ff, bias=False)
121
+ self.down = nn.Linear(cfg.d_ff, cfg.d_model, bias=False)
122
+
123
+ def forward(self, x):
124
+ return self.down(F.silu(self.gate(x)) * self.up(x))
125
+
126
+
127
+ class Block(nn.Module):
128
+ def __init__(self, cfg: CenserConfig) -> None:
129
+ super().__init__()
130
+ self.norm1 = RMSNorm(cfg.d_model)
131
+ self.attn = Attention(cfg)
132
+ self.norm2 = RMSNorm(cfg.d_model)
133
+ self.mlp = SwiGLU(cfg)
134
+ self.dropout = nn.Dropout(cfg.dropout)
135
+
136
+ def forward(self, x, cos, sin, mask):
137
+ x = x + self.dropout(self.attn(self.norm1(x), cos, sin, mask))
138
+ x = x + self.dropout(self.mlp(self.norm2(x)))
139
+ return x
140
+
141
+
142
+ class CenserForSequenceClassification(PreTrainedModel):
143
+ """Transformer trunk plus a two-way answer head: logits [safe, unsafe]."""
144
+
145
+ config_class = CenserConfig
146
+ base_model_prefix = ""
147
+ main_input_name = "input_ids"
148
+
149
+ def __init__(self, cfg: CenserConfig) -> None:
150
+ super().__init__(cfg)
151
+ self.cfg = cfg
152
+ self.embed = nn.Embedding(cfg.vocab_size, cfg.d_model, padding_idx=cfg.pad_id)
153
+ self.blocks = nn.ModuleList([Block(cfg) for _ in range(cfg.n_layers)])
154
+ self.norm = RMSNorm(cfg.d_model)
155
+ self.lm_head = nn.Linear(cfg.d_model, cfg.vocab_size, bias=False)
156
+ self._rope_cache = None
157
+ self.post_init()
158
+
159
+ def _rope(self, device, dtype):
160
+ key = f"{device}|{dtype}"
161
+ if self._rope_cache is None or self._rope_cache[2] != key:
162
+ head_dim = self.cfg.d_model // self.cfg.n_heads
163
+ cos, sin = build_rope_cache(self.cfg.max_seq_len, head_dim, device, dtype)
164
+ self._rope_cache = (cos, sin, key)
165
+ return self._rope_cache[0], self._rope_cache[1]
166
+
167
+ def _attention_mask(self, input_ids):
168
+ t = input_ids.shape[1]
169
+ pad = input_ids != self.cfg.pad_id
170
+ causal = torch.ones(t, t, dtype=torch.bool, device=input_ids.device).tril()
171
+ return pad[:, None, None, :] & causal[None, None, :, :]
172
+
173
+ def forward(self, input_ids=None, attention_mask=None, labels=None, **kwargs):
174
+ if input_ids is None:
175
+ raise ValueError("input_ids is required")
176
+ cos, sin = self._rope(input_ids.device, self.embed.weight.dtype)
177
+ mask = self._attention_mask(input_ids)
178
+ x = self.embed(input_ids)
179
+ for block in self.blocks:
180
+ x = block(x, cos, sin, mask)
181
+ hidden = self.norm(x)
182
+ non_pad = (input_ids != self.cfg.pad_id).sum(dim=1).clamp(min=1)
183
+ idx = (non_pad - 1)[:, None, None].expand(-1, 1, hidden.shape[-1])
184
+ pooled = hidden.gather(1, idx)[:, 0, :]
185
+ logits = self.lm_head(pooled).float()
186
+ answer_ids = torch.tensor(self.cfg.answer_ids, device=logits.device)
187
+ logits = logits.index_select(-1, answer_ids)
188
+
189
+ loss = None
190
+ if labels is not None:
191
+ loss = F.cross_entropy(logits, labels.to(logits.device))
192
+ return SequenceClassifierOutput(logits=logits, loss=loss)
tokenizer.json CHANGED
@@ -50,11 +50,29 @@
50
  "post_processor": {
51
  "type": "TemplateProcessing",
52
  "single": [
 
 
 
 
 
 
 
 
 
 
 
 
53
  {
54
  "Sequence": {
55
  "id": "A",
56
  "type_id": 0
57
  }
 
 
 
 
 
 
58
  }
59
  ],
60
  "pair": [
@@ -72,6 +90,15 @@
72
  }
73
  ],
74
  "special_tokens": {
 
 
 
 
 
 
 
 
 
75
  "[BOS]": {
76
  "id": "[BOS]",
77
  "ids": [
@@ -81,13 +108,13 @@
81
  "[BOS]"
82
  ]
83
  },
84
- "[PAD]": {
85
- "id": "[PAD]",
86
  "ids": [
87
- 0
88
  ],
89
  "tokens": [
90
- "[PAD]"
91
  ]
92
  }
93
  }
 
50
  "post_processor": {
51
  "type": "TemplateProcessing",
52
  "single": [
53
+ {
54
+ "SpecialToken": {
55
+ "id": "[BOS]",
56
+ "type_id": 0
57
+ }
58
+ },
59
+ {
60
+ "SpecialToken": {
61
+ "id": "[USER]",
62
+ "type_id": 0
63
+ }
64
+ },
65
  {
66
  "Sequence": {
67
  "id": "A",
68
  "type_id": 0
69
  }
70
+ },
71
+ {
72
+ "SpecialToken": {
73
+ "id": "[ASSISTANT]",
74
+ "type_id": 0
75
+ }
76
  }
77
  ],
78
  "pair": [
 
90
  }
91
  ],
92
  "special_tokens": {
93
+ "[ASSISTANT]": {
94
+ "id": "[ASSISTANT]",
95
+ "ids": [
96
+ 3
97
+ ],
98
+ "tokens": [
99
+ "[ASSISTANT]"
100
+ ]
101
+ },
102
  "[BOS]": {
103
  "id": "[BOS]",
104
  "ids": [
 
108
  "[BOS]"
109
  ]
110
  },
111
+ "[USER]": {
112
+ "id": "[USER]",
113
  "ids": [
114
+ 2
115
  ],
116
  "tokens": [
117
+ "[USER]"
118
  ]
119
  }
120
  }