KucLab commited on
Commit
9da4e93
·
verified ·
1 Parent(s): fb17ab1

Upload Censer.1

Browse files
Files changed (4) hide show
  1. README.md +37 -6
  2. censer.py +225 -0
  3. config.json +28 -0
  4. model.safetensors +3 -0
README.md CHANGED
@@ -103,16 +103,47 @@ Training the whole model from scratch takes about 3.5 minutes on the same GPU.
103
 
104
  ## Usage
105
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
106
  ```python
107
- import torch
108
- from kuclab.infer import SafetyModel
109
 
110
- model = SafetyModel("checkpoints/best.pt")
111
- verdict = model.predict("how to make a bomb at home")
112
- print(verdict.label) # "1"
113
- print(verdict.probability) # 0.886
 
114
  ```
115
 
 
 
 
 
 
 
 
 
 
 
 
 
116
  ## Limitations
117
 
118
  - **English only.** Other languages mostly score `0`.
 
103
 
104
  ## Usage
105
 
106
+ This repository is self-contained. You do not need to clone the training code.
107
+
108
+ ```bash
109
+ pip install torch tokenizers safetensors huggingface_hub
110
+
111
+ python - <<'PY'
112
+ from huggingface_hub import snapshot_download
113
+ snapshot_download("KucLab/Censer.1", local_dir="Censer.1")
114
+ PY
115
+
116
+ cd Censer.1
117
+ python censer.py "how to make a bomb at home" # -> 1
118
+ python censer.py "Hello how are you?" # -> 0
119
+ python censer.py "suck my dick" --json
120
+ # {"label": "1", "unsafe": true, "score": 0.9944}
121
+ ```
122
+
123
+ Or use the same file as a library:
124
+
125
  ```python
126
+ from censer import CenserPipeline
 
127
 
128
+ pipe = CenserPipeline("Censer.1")
129
+ print(pipe.predict("how to make a bomb at home"))
130
+ # {"label": "1", "unsafe": True, "score": 0.8167}
131
+
132
+ print(pipe.predict_batch(["Hello how are you?", "suck my dick"]))
133
  ```
134
 
135
+ Files:
136
+
137
+ | File | Purpose |
138
+ | --- | --- |
139
+ | `model.safetensors` | weights only, 4.3 MB |
140
+ | `config.json` | architecture and inference settings |
141
+ | `tokenizer.json` | byte-level BPE, 2048 tokens |
142
+ | `censer.py` | standalone model + inference, no repo needed |
143
+
144
+ `censer.py` was verified to produce byte-identical predictions to the training
145
+ code. Runs on CPU or CUDA/ROCm; pass `device="cpu"` to force CPU.
146
+
147
  ## Limitations
148
 
149
  - **English only.** Other languages mostly score `0`.
censer.py ADDED
@@ -0,0 +1,225 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Inference for Censer.1 — self-contained, no dependency on the training repo.
2
+
3
+ Copy this file next to the downloaded weights, install torch and tokenizers,
4
+ and you are ready:
5
+
6
+ pip install torch tokenizers huggingface_hub
7
+ python censer.py "how to make a bomb at home"
8
+
9
+ It expects, in the same directory:
10
+ model.safetensors
11
+ tokenizer.json
12
+ config.json
13
+ """
14
+
15
+ from __future__ import annotations
16
+
17
+ import json
18
+ import math
19
+ from pathlib import Path
20
+
21
+ import torch
22
+ import torch.nn as nn
23
+ import torch.nn.functional as F
24
+ from tokenizers import Tokenizer
25
+
26
+ PAD, BOS, USER, ASSISTANT = "[PAD]", "[BOS]", "[USER]", "[ASSISTANT]"
27
+
28
+
29
+ class RMSNorm(nn.Module):
30
+ def __init__(self, dim: int, eps: float = 1e-5) -> None:
31
+ super().__init__()
32
+ self.eps = eps
33
+ self.weight = nn.Parameter(torch.ones(dim))
34
+
35
+ def forward(self, x: torch.Tensor) -> torch.Tensor:
36
+ dtype = x.dtype
37
+ x = x.float()
38
+ x = x * torch.rsqrt(x.pow(2).mean(-1, keepdim=True) + self.eps)
39
+ return x.to(dtype) * self.weight
40
+
41
+
42
+ def build_rope_cache(seq_len: int, head_dim: int, device, dtype, base: float = 10000.0):
43
+ half = head_dim // 2
44
+ freqs = 1.0 / (base ** (torch.arange(0, half, device=device, dtype=torch.float32) / half))
45
+ positions = torch.arange(seq_len, device=device, dtype=torch.float32)
46
+ angles = torch.outer(positions, freqs)
47
+ cos = torch.cat((angles.cos(), angles.cos()), dim=-1).to(dtype)[:, None, :]
48
+ sin = torch.cat((angles.sin(), angles.sin()), dim=-1).to(dtype)[:, None, :]
49
+ return cos, sin
50
+
51
+
52
+ def apply_rope(x: torch.Tensor, cos: torch.Tensor, sin: torch.Tensor) -> torch.Tensor:
53
+ seq_len = x.shape[1]
54
+ cos, sin = cos[:seq_len], sin[:seq_len]
55
+ x1, x2 = x[..., : x.shape[-1] // 2], x[..., x.shape[-1] // 2 :]
56
+ rotated = torch.cat((-x2, x1), dim=-1)
57
+ return x * cos + rotated * sin
58
+
59
+
60
+ class Attention(nn.Module):
61
+ def __init__(self, cfg: dict) -> None:
62
+ super().__init__()
63
+ self.n_heads = cfg["n_heads"]
64
+ self.head_dim = cfg["d_model"] // cfg["n_heads"]
65
+ self.dropout = cfg["dropout"]
66
+ self.qkv = nn.Linear(cfg["d_model"], 3 * cfg["d_model"], bias=False)
67
+ self.proj = nn.Linear(cfg["d_model"], cfg["d_model"], bias=False)
68
+
69
+ def forward(self, x, cos, sin, mask):
70
+ b, t, _ = x.shape
71
+ q, k, v = self.qkv(x).split(x.shape[-1], dim=-1)
72
+ q = q.view(b, t, self.n_heads, self.head_dim)
73
+ k = k.view(b, t, self.n_heads, self.head_dim)
74
+ v = v.view(b, t, self.n_heads, self.head_dim)
75
+ q, k = apply_rope(q, cos, sin), apply_rope(k, cos, sin)
76
+ out = F.scaled_dot_product_attention(
77
+ q.transpose(1, 2), k.transpose(1, 2), v.transpose(1, 2),
78
+ attn_mask=mask,
79
+ dropout_p=self.dropout if self.training else 0.0,
80
+ is_causal=mask is None,
81
+ )
82
+ return self.proj(out.transpose(1, 2).reshape(b, t, -1))
83
+
84
+
85
+ class SwiGLU(nn.Module):
86
+ def __init__(self, cfg: dict) -> None:
87
+ super().__init__()
88
+ self.gate = nn.Linear(cfg["d_model"], cfg["d_ff"], bias=False)
89
+ self.up = nn.Linear(cfg["d_model"], cfg["d_ff"], bias=False)
90
+ self.down = nn.Linear(cfg["d_ff"], cfg["d_model"], bias=False)
91
+
92
+ def forward(self, x):
93
+ return self.down(F.silu(self.gate(x)) * self.up(x))
94
+
95
+
96
+ class Block(nn.Module):
97
+ def __init__(self, cfg: dict) -> None:
98
+ super().__init__()
99
+ self.norm1 = RMSNorm(cfg["d_model"])
100
+ self.attn = Attention(cfg)
101
+ self.norm2 = RMSNorm(cfg["d_model"])
102
+ self.mlp = SwiGLU(cfg)
103
+ self.dropout = nn.Dropout(cfg["dropout"])
104
+
105
+ def forward(self, x, cos, sin, mask):
106
+ x = x + self.dropout(self.attn(self.norm1(x), cos, sin, mask))
107
+ x = x + self.dropout(self.mlp(self.norm2(x)))
108
+ return x
109
+
110
+
111
+ class Censer(nn.Module):
112
+ def __init__(self, cfg: dict) -> None:
113
+ super().__init__()
114
+ self.cfg = cfg
115
+ self.register_buffer(
116
+ "answer_id_tensor", torch.tensor(cfg["answer_ids"], dtype=torch.long), persistent=False
117
+ )
118
+ self.embed = nn.Embedding(cfg["vocab_size"], cfg["d_model"], padding_idx=cfg["pad_id"])
119
+ self.blocks = nn.ModuleList([Block(cfg) for _ in range(cfg["n_layers"])])
120
+ self.norm = RMSNorm(cfg["d_model"])
121
+ self.lm_head = nn.Linear(cfg["d_model"], cfg["vocab_size"], bias=False)
122
+ if cfg["tie_embeddings"]:
123
+ self.lm_head.weight = self.embed.weight
124
+ self._rope_cache = None
125
+
126
+ def _rope(self, device, dtype):
127
+ key = f"{device}|{dtype}"
128
+ if self._rope_cache is None or self._rope_cache[2] != key:
129
+ head_dim = self.cfg["d_model"] // self.cfg["n_heads"]
130
+ cos, sin = build_rope_cache(self.cfg["max_seq_len"], head_dim, device, dtype)
131
+ self._rope_cache = (cos, sin, key)
132
+ return self._rope_cache[0], self._rope_cache[1]
133
+
134
+ def _attention_mask(self, input_ids):
135
+ b, t = input_ids.shape
136
+ pad = input_ids != self.cfg["pad_id"]
137
+ causal = torch.ones(t, t, dtype=torch.bool, device=input_ids.device).tril()
138
+ return pad[:, None, None, :] & causal[None, None, :, :]
139
+
140
+ def forward(self, input_ids):
141
+ cos, sin = self._rope(input_ids.device, self.embed.weight.dtype)
142
+ mask = self._attention_mask(input_ids)
143
+ x = self.embed(input_ids)
144
+ for block in self.blocks:
145
+ x = block(x, cos, sin, mask)
146
+ return self.lm_head(self.norm(x))
147
+
148
+ @torch.no_grad()
149
+ def classify(self, input_ids):
150
+ hidden = self.forward(input_ids)
151
+ non_pad = (input_ids != self.cfg["pad_id"]).sum(dim=1).clamp(min=1)
152
+ idx = (non_pad - 1)[:, None, None].expand(-1, 1, hidden.shape[-1])
153
+ logits = hidden.gather(1, idx)[:, 0, :].float()
154
+ logits = logits.index_select(-1, self.answer_id_tensor)
155
+ return logits.softmax(-1)[:, 1]
156
+
157
+
158
+ class CenserPipeline:
159
+ """Text in, "1" (unsafe) or "0" (safe) out."""
160
+
161
+ def __init__(self, directory: str | Path = ".", threshold: float = 0.5, device: str | None = None) -> None:
162
+ from safetensors.torch import load_file
163
+
164
+ directory = Path(directory)
165
+ self.cfg = json.loads((directory / "config.json").read_text(encoding="utf-8"))
166
+ self.tok = Tokenizer.from_file(str(directory / "tokenizer.json"))
167
+ self.pad_id = self.tok.token_to_id(PAD)
168
+ self.bos_id = self.tok.token_to_id(BOS)
169
+ self.user_id = self.tok.token_to_id(USER)
170
+ self.assistant_id = self.tok.token_to_id(ASSISTANT)
171
+ self.threshold = threshold
172
+ self.device = torch.device(device or ("cuda" if torch.cuda.is_available() else "cpu"))
173
+ self.model = Censer(self.cfg)
174
+ state = load_file(directory / "model.safetensors")
175
+ if self.cfg["tie_embeddings"]:
176
+ state.pop("lm_head.weight", None)
177
+ missing, unexpected = self.model.load_state_dict(state, strict=False)
178
+ missing = [k for k in missing if not (self.cfg["tie_embeddings"] and k == "lm_head.weight")]
179
+ if missing or unexpected:
180
+ raise RuntimeError(f"weight mismatch: missing={missing} unexpected={unexpected}")
181
+ self.model.to(self.device).eval()
182
+ self.max_chars = self.cfg["max_chars"]
183
+
184
+ def _encode(self, text: str) -> list[int]:
185
+ body = self.tok.encode(text[: self.max_chars], add_special_tokens=False).ids
186
+ ids = [self.bos_id, self.user_id, *body, self.assistant_id]
187
+ return ids[: self.cfg["max_seq_len"]]
188
+
189
+ def predict(self, text: str) -> dict:
190
+ return self.predict_batch([text])[0]
191
+
192
+ def predict_batch(self, texts: list[str]) -> list[dict]:
193
+ seqs = [self._encode(t) for t in texts]
194
+ width = max(len(s) for s in seqs)
195
+ ids = torch.full((len(seqs), width), self.pad_id, dtype=torch.long)
196
+ for i, seq in enumerate(seqs):
197
+ ids[i, : len(seq)] = torch.tensor(seq)
198
+ probs = self.model.classify(ids.to(self.device)).float().cpu().tolist()
199
+ return [
200
+ {"label": "1" if p >= self.threshold else "0", "unsafe": p >= self.threshold, "score": round(p, 4)}
201
+ for p in probs
202
+ ]
203
+
204
+
205
+ def main() -> None:
206
+ import argparse
207
+
208
+ parser = argparse.ArgumentParser(description="Censer.1 safety classifier.")
209
+ parser.add_argument("text", nargs="*")
210
+ parser.add_argument("--model-dir", default=".")
211
+ parser.add_argument("--threshold", type=float, default=0.5)
212
+ parser.add_argument("--json", action="store_true")
213
+ args = parser.parse_args()
214
+
215
+ pipe = CenserPipeline(args.model_dir, args.threshold)
216
+ text = " ".join(args.text).strip()
217
+ if not text:
218
+ print("usage: python censer.py \"your text\"")
219
+ raise SystemExit(2)
220
+ verdict = pipe.predict(text)
221
+ print(json.dumps(verdict) if args.json else verdict["label"])
222
+
223
+
224
+ if __name__ == "__main__":
225
+ main()
config.json ADDED
@@ -0,0 +1,28 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "architectures": [
3
+ "Censer"
4
+ ],
5
+ "model_type": "censer",
6
+ "name": "Censer.1",
7
+ "vocab_size": 2048,
8
+ "n_layers": 4,
9
+ "n_heads": 4,
10
+ "d_model": 128,
11
+ "d_ff": 384,
12
+ "max_seq_len": 160,
13
+ "dropout": 0.1,
14
+ "tie_embeddings": true,
15
+ "pad_id": 0,
16
+ "max_chars": 500,
17
+ "answer_ids": [
18
+ 19,
19
+ 20
20
+ ],
21
+ "special_tokens": [
22
+ "[PAD]",
23
+ "[BOS]",
24
+ "[USER]",
25
+ "[ASSISTANT]"
26
+ ],
27
+ "threshold": 0.5
28
+ }
model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e43803a73dd3e9bdf4e510fd33af5a07e87fb5cfef3dc38d62cc383c57e85948
3
+ size 4463808