Text Classification
Transformers
Safetensors
English
llama
safety
content-moderation
tiny-language-model
from-scratch
text-embeddings-inference
Instructions to use KucLab/Censer.1 with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- Transformers
How to use KucLab/Censer.1 with Transformers:
# Use a pipeline as a high-level helper from transformers import pipeline pipe = pipeline("text-classification", model="KucLab/Censer.1")# pip install -U transformers accelerate # Load model directly from transformers import AutoTokenizer, AutoModelForSequenceClassification tokenizer = AutoTokenizer.from_pretrained("KucLab/Censer.1") model = AutoModelForSequenceClassification.from_pretrained("KucLab/Censer.1", device_map="auto") - Notebooks
- Google Colab
- Kaggle
Upload Censer.1
Browse files- README.md +37 -6
- censer.py +225 -0
- config.json +28 -0
- model.safetensors +3 -0
README.md
CHANGED
|
@@ -103,16 +103,47 @@ Training the whole model from scratch takes about 3.5 minutes on the same GPU.
|
|
| 103 |
|
| 104 |
## Usage
|
| 105 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 106 |
```python
|
| 107 |
-
import
|
| 108 |
-
from kuclab.infer import SafetyModel
|
| 109 |
|
| 110 |
-
|
| 111 |
-
|
| 112 |
-
|
| 113 |
-
|
|
|
|
| 114 |
```
|
| 115 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 116 |
## Limitations
|
| 117 |
|
| 118 |
- **English only.** Other languages mostly score `0`.
|
|
|
|
| 103 |
|
| 104 |
## Usage
|
| 105 |
|
| 106 |
+
This repository is self-contained. You do not need to clone the training code.
|
| 107 |
+
|
| 108 |
+
```bash
|
| 109 |
+
pip install torch tokenizers safetensors huggingface_hub
|
| 110 |
+
|
| 111 |
+
python - <<'PY'
|
| 112 |
+
from huggingface_hub import snapshot_download
|
| 113 |
+
snapshot_download("KucLab/Censer.1", local_dir="Censer.1")
|
| 114 |
+
PY
|
| 115 |
+
|
| 116 |
+
cd Censer.1
|
| 117 |
+
python censer.py "how to make a bomb at home" # -> 1
|
| 118 |
+
python censer.py "Hello how are you?" # -> 0
|
| 119 |
+
python censer.py "suck my dick" --json
|
| 120 |
+
# {"label": "1", "unsafe": true, "score": 0.9944}
|
| 121 |
+
```
|
| 122 |
+
|
| 123 |
+
Or use the same file as a library:
|
| 124 |
+
|
| 125 |
```python
|
| 126 |
+
from censer import CenserPipeline
|
|
|
|
| 127 |
|
| 128 |
+
pipe = CenserPipeline("Censer.1")
|
| 129 |
+
print(pipe.predict("how to make a bomb at home"))
|
| 130 |
+
# {"label": "1", "unsafe": True, "score": 0.8167}
|
| 131 |
+
|
| 132 |
+
print(pipe.predict_batch(["Hello how are you?", "suck my dick"]))
|
| 133 |
```
|
| 134 |
|
| 135 |
+
Files:
|
| 136 |
+
|
| 137 |
+
| File | Purpose |
|
| 138 |
+
| --- | --- |
|
| 139 |
+
| `model.safetensors` | weights only, 4.3 MB |
|
| 140 |
+
| `config.json` | architecture and inference settings |
|
| 141 |
+
| `tokenizer.json` | byte-level BPE, 2048 tokens |
|
| 142 |
+
| `censer.py` | standalone model + inference, no repo needed |
|
| 143 |
+
|
| 144 |
+
`censer.py` was verified to produce byte-identical predictions to the training
|
| 145 |
+
code. Runs on CPU or CUDA/ROCm; pass `device="cpu"` to force CPU.
|
| 146 |
+
|
| 147 |
## Limitations
|
| 148 |
|
| 149 |
- **English only.** Other languages mostly score `0`.
|
censer.py
ADDED
|
@@ -0,0 +1,225 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Inference for Censer.1 — self-contained, no dependency on the training repo.
|
| 2 |
+
|
| 3 |
+
Copy this file next to the downloaded weights, install torch and tokenizers,
|
| 4 |
+
and you are ready:
|
| 5 |
+
|
| 6 |
+
pip install torch tokenizers huggingface_hub
|
| 7 |
+
python censer.py "how to make a bomb at home"
|
| 8 |
+
|
| 9 |
+
It expects, in the same directory:
|
| 10 |
+
model.safetensors
|
| 11 |
+
tokenizer.json
|
| 12 |
+
config.json
|
| 13 |
+
"""
|
| 14 |
+
|
| 15 |
+
from __future__ import annotations
|
| 16 |
+
|
| 17 |
+
import json
|
| 18 |
+
import math
|
| 19 |
+
from pathlib import Path
|
| 20 |
+
|
| 21 |
+
import torch
|
| 22 |
+
import torch.nn as nn
|
| 23 |
+
import torch.nn.functional as F
|
| 24 |
+
from tokenizers import Tokenizer
|
| 25 |
+
|
| 26 |
+
PAD, BOS, USER, ASSISTANT = "[PAD]", "[BOS]", "[USER]", "[ASSISTANT]"
|
| 27 |
+
|
| 28 |
+
|
| 29 |
+
class RMSNorm(nn.Module):
|
| 30 |
+
def __init__(self, dim: int, eps: float = 1e-5) -> None:
|
| 31 |
+
super().__init__()
|
| 32 |
+
self.eps = eps
|
| 33 |
+
self.weight = nn.Parameter(torch.ones(dim))
|
| 34 |
+
|
| 35 |
+
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
| 36 |
+
dtype = x.dtype
|
| 37 |
+
x = x.float()
|
| 38 |
+
x = x * torch.rsqrt(x.pow(2).mean(-1, keepdim=True) + self.eps)
|
| 39 |
+
return x.to(dtype) * self.weight
|
| 40 |
+
|
| 41 |
+
|
| 42 |
+
def build_rope_cache(seq_len: int, head_dim: int, device, dtype, base: float = 10000.0):
|
| 43 |
+
half = head_dim // 2
|
| 44 |
+
freqs = 1.0 / (base ** (torch.arange(0, half, device=device, dtype=torch.float32) / half))
|
| 45 |
+
positions = torch.arange(seq_len, device=device, dtype=torch.float32)
|
| 46 |
+
angles = torch.outer(positions, freqs)
|
| 47 |
+
cos = torch.cat((angles.cos(), angles.cos()), dim=-1).to(dtype)[:, None, :]
|
| 48 |
+
sin = torch.cat((angles.sin(), angles.sin()), dim=-1).to(dtype)[:, None, :]
|
| 49 |
+
return cos, sin
|
| 50 |
+
|
| 51 |
+
|
| 52 |
+
def apply_rope(x: torch.Tensor, cos: torch.Tensor, sin: torch.Tensor) -> torch.Tensor:
|
| 53 |
+
seq_len = x.shape[1]
|
| 54 |
+
cos, sin = cos[:seq_len], sin[:seq_len]
|
| 55 |
+
x1, x2 = x[..., : x.shape[-1] // 2], x[..., x.shape[-1] // 2 :]
|
| 56 |
+
rotated = torch.cat((-x2, x1), dim=-1)
|
| 57 |
+
return x * cos + rotated * sin
|
| 58 |
+
|
| 59 |
+
|
| 60 |
+
class Attention(nn.Module):
|
| 61 |
+
def __init__(self, cfg: dict) -> None:
|
| 62 |
+
super().__init__()
|
| 63 |
+
self.n_heads = cfg["n_heads"]
|
| 64 |
+
self.head_dim = cfg["d_model"] // cfg["n_heads"]
|
| 65 |
+
self.dropout = cfg["dropout"]
|
| 66 |
+
self.qkv = nn.Linear(cfg["d_model"], 3 * cfg["d_model"], bias=False)
|
| 67 |
+
self.proj = nn.Linear(cfg["d_model"], cfg["d_model"], bias=False)
|
| 68 |
+
|
| 69 |
+
def forward(self, x, cos, sin, mask):
|
| 70 |
+
b, t, _ = x.shape
|
| 71 |
+
q, k, v = self.qkv(x).split(x.shape[-1], dim=-1)
|
| 72 |
+
q = q.view(b, t, self.n_heads, self.head_dim)
|
| 73 |
+
k = k.view(b, t, self.n_heads, self.head_dim)
|
| 74 |
+
v = v.view(b, t, self.n_heads, self.head_dim)
|
| 75 |
+
q, k = apply_rope(q, cos, sin), apply_rope(k, cos, sin)
|
| 76 |
+
out = F.scaled_dot_product_attention(
|
| 77 |
+
q.transpose(1, 2), k.transpose(1, 2), v.transpose(1, 2),
|
| 78 |
+
attn_mask=mask,
|
| 79 |
+
dropout_p=self.dropout if self.training else 0.0,
|
| 80 |
+
is_causal=mask is None,
|
| 81 |
+
)
|
| 82 |
+
return self.proj(out.transpose(1, 2).reshape(b, t, -1))
|
| 83 |
+
|
| 84 |
+
|
| 85 |
+
class SwiGLU(nn.Module):
|
| 86 |
+
def __init__(self, cfg: dict) -> None:
|
| 87 |
+
super().__init__()
|
| 88 |
+
self.gate = nn.Linear(cfg["d_model"], cfg["d_ff"], bias=False)
|
| 89 |
+
self.up = nn.Linear(cfg["d_model"], cfg["d_ff"], bias=False)
|
| 90 |
+
self.down = nn.Linear(cfg["d_ff"], cfg["d_model"], bias=False)
|
| 91 |
+
|
| 92 |
+
def forward(self, x):
|
| 93 |
+
return self.down(F.silu(self.gate(x)) * self.up(x))
|
| 94 |
+
|
| 95 |
+
|
| 96 |
+
class Block(nn.Module):
|
| 97 |
+
def __init__(self, cfg: dict) -> None:
|
| 98 |
+
super().__init__()
|
| 99 |
+
self.norm1 = RMSNorm(cfg["d_model"])
|
| 100 |
+
self.attn = Attention(cfg)
|
| 101 |
+
self.norm2 = RMSNorm(cfg["d_model"])
|
| 102 |
+
self.mlp = SwiGLU(cfg)
|
| 103 |
+
self.dropout = nn.Dropout(cfg["dropout"])
|
| 104 |
+
|
| 105 |
+
def forward(self, x, cos, sin, mask):
|
| 106 |
+
x = x + self.dropout(self.attn(self.norm1(x), cos, sin, mask))
|
| 107 |
+
x = x + self.dropout(self.mlp(self.norm2(x)))
|
| 108 |
+
return x
|
| 109 |
+
|
| 110 |
+
|
| 111 |
+
class Censer(nn.Module):
|
| 112 |
+
def __init__(self, cfg: dict) -> None:
|
| 113 |
+
super().__init__()
|
| 114 |
+
self.cfg = cfg
|
| 115 |
+
self.register_buffer(
|
| 116 |
+
"answer_id_tensor", torch.tensor(cfg["answer_ids"], dtype=torch.long), persistent=False
|
| 117 |
+
)
|
| 118 |
+
self.embed = nn.Embedding(cfg["vocab_size"], cfg["d_model"], padding_idx=cfg["pad_id"])
|
| 119 |
+
self.blocks = nn.ModuleList([Block(cfg) for _ in range(cfg["n_layers"])])
|
| 120 |
+
self.norm = RMSNorm(cfg["d_model"])
|
| 121 |
+
self.lm_head = nn.Linear(cfg["d_model"], cfg["vocab_size"], bias=False)
|
| 122 |
+
if cfg["tie_embeddings"]:
|
| 123 |
+
self.lm_head.weight = self.embed.weight
|
| 124 |
+
self._rope_cache = None
|
| 125 |
+
|
| 126 |
+
def _rope(self, device, dtype):
|
| 127 |
+
key = f"{device}|{dtype}"
|
| 128 |
+
if self._rope_cache is None or self._rope_cache[2] != key:
|
| 129 |
+
head_dim = self.cfg["d_model"] // self.cfg["n_heads"]
|
| 130 |
+
cos, sin = build_rope_cache(self.cfg["max_seq_len"], head_dim, device, dtype)
|
| 131 |
+
self._rope_cache = (cos, sin, key)
|
| 132 |
+
return self._rope_cache[0], self._rope_cache[1]
|
| 133 |
+
|
| 134 |
+
def _attention_mask(self, input_ids):
|
| 135 |
+
b, t = input_ids.shape
|
| 136 |
+
pad = input_ids != self.cfg["pad_id"]
|
| 137 |
+
causal = torch.ones(t, t, dtype=torch.bool, device=input_ids.device).tril()
|
| 138 |
+
return pad[:, None, None, :] & causal[None, None, :, :]
|
| 139 |
+
|
| 140 |
+
def forward(self, input_ids):
|
| 141 |
+
cos, sin = self._rope(input_ids.device, self.embed.weight.dtype)
|
| 142 |
+
mask = self._attention_mask(input_ids)
|
| 143 |
+
x = self.embed(input_ids)
|
| 144 |
+
for block in self.blocks:
|
| 145 |
+
x = block(x, cos, sin, mask)
|
| 146 |
+
return self.lm_head(self.norm(x))
|
| 147 |
+
|
| 148 |
+
@torch.no_grad()
|
| 149 |
+
def classify(self, input_ids):
|
| 150 |
+
hidden = self.forward(input_ids)
|
| 151 |
+
non_pad = (input_ids != self.cfg["pad_id"]).sum(dim=1).clamp(min=1)
|
| 152 |
+
idx = (non_pad - 1)[:, None, None].expand(-1, 1, hidden.shape[-1])
|
| 153 |
+
logits = hidden.gather(1, idx)[:, 0, :].float()
|
| 154 |
+
logits = logits.index_select(-1, self.answer_id_tensor)
|
| 155 |
+
return logits.softmax(-1)[:, 1]
|
| 156 |
+
|
| 157 |
+
|
| 158 |
+
class CenserPipeline:
|
| 159 |
+
"""Text in, "1" (unsafe) or "0" (safe) out."""
|
| 160 |
+
|
| 161 |
+
def __init__(self, directory: str | Path = ".", threshold: float = 0.5, device: str | None = None) -> None:
|
| 162 |
+
from safetensors.torch import load_file
|
| 163 |
+
|
| 164 |
+
directory = Path(directory)
|
| 165 |
+
self.cfg = json.loads((directory / "config.json").read_text(encoding="utf-8"))
|
| 166 |
+
self.tok = Tokenizer.from_file(str(directory / "tokenizer.json"))
|
| 167 |
+
self.pad_id = self.tok.token_to_id(PAD)
|
| 168 |
+
self.bos_id = self.tok.token_to_id(BOS)
|
| 169 |
+
self.user_id = self.tok.token_to_id(USER)
|
| 170 |
+
self.assistant_id = self.tok.token_to_id(ASSISTANT)
|
| 171 |
+
self.threshold = threshold
|
| 172 |
+
self.device = torch.device(device or ("cuda" if torch.cuda.is_available() else "cpu"))
|
| 173 |
+
self.model = Censer(self.cfg)
|
| 174 |
+
state = load_file(directory / "model.safetensors")
|
| 175 |
+
if self.cfg["tie_embeddings"]:
|
| 176 |
+
state.pop("lm_head.weight", None)
|
| 177 |
+
missing, unexpected = self.model.load_state_dict(state, strict=False)
|
| 178 |
+
missing = [k for k in missing if not (self.cfg["tie_embeddings"] and k == "lm_head.weight")]
|
| 179 |
+
if missing or unexpected:
|
| 180 |
+
raise RuntimeError(f"weight mismatch: missing={missing} unexpected={unexpected}")
|
| 181 |
+
self.model.to(self.device).eval()
|
| 182 |
+
self.max_chars = self.cfg["max_chars"]
|
| 183 |
+
|
| 184 |
+
def _encode(self, text: str) -> list[int]:
|
| 185 |
+
body = self.tok.encode(text[: self.max_chars], add_special_tokens=False).ids
|
| 186 |
+
ids = [self.bos_id, self.user_id, *body, self.assistant_id]
|
| 187 |
+
return ids[: self.cfg["max_seq_len"]]
|
| 188 |
+
|
| 189 |
+
def predict(self, text: str) -> dict:
|
| 190 |
+
return self.predict_batch([text])[0]
|
| 191 |
+
|
| 192 |
+
def predict_batch(self, texts: list[str]) -> list[dict]:
|
| 193 |
+
seqs = [self._encode(t) for t in texts]
|
| 194 |
+
width = max(len(s) for s in seqs)
|
| 195 |
+
ids = torch.full((len(seqs), width), self.pad_id, dtype=torch.long)
|
| 196 |
+
for i, seq in enumerate(seqs):
|
| 197 |
+
ids[i, : len(seq)] = torch.tensor(seq)
|
| 198 |
+
probs = self.model.classify(ids.to(self.device)).float().cpu().tolist()
|
| 199 |
+
return [
|
| 200 |
+
{"label": "1" if p >= self.threshold else "0", "unsafe": p >= self.threshold, "score": round(p, 4)}
|
| 201 |
+
for p in probs
|
| 202 |
+
]
|
| 203 |
+
|
| 204 |
+
|
| 205 |
+
def main() -> None:
|
| 206 |
+
import argparse
|
| 207 |
+
|
| 208 |
+
parser = argparse.ArgumentParser(description="Censer.1 safety classifier.")
|
| 209 |
+
parser.add_argument("text", nargs="*")
|
| 210 |
+
parser.add_argument("--model-dir", default=".")
|
| 211 |
+
parser.add_argument("--threshold", type=float, default=0.5)
|
| 212 |
+
parser.add_argument("--json", action="store_true")
|
| 213 |
+
args = parser.parse_args()
|
| 214 |
+
|
| 215 |
+
pipe = CenserPipeline(args.model_dir, args.threshold)
|
| 216 |
+
text = " ".join(args.text).strip()
|
| 217 |
+
if not text:
|
| 218 |
+
print("usage: python censer.py \"your text\"")
|
| 219 |
+
raise SystemExit(2)
|
| 220 |
+
verdict = pipe.predict(text)
|
| 221 |
+
print(json.dumps(verdict) if args.json else verdict["label"])
|
| 222 |
+
|
| 223 |
+
|
| 224 |
+
if __name__ == "__main__":
|
| 225 |
+
main()
|
config.json
ADDED
|
@@ -0,0 +1,28 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"architectures": [
|
| 3 |
+
"Censer"
|
| 4 |
+
],
|
| 5 |
+
"model_type": "censer",
|
| 6 |
+
"name": "Censer.1",
|
| 7 |
+
"vocab_size": 2048,
|
| 8 |
+
"n_layers": 4,
|
| 9 |
+
"n_heads": 4,
|
| 10 |
+
"d_model": 128,
|
| 11 |
+
"d_ff": 384,
|
| 12 |
+
"max_seq_len": 160,
|
| 13 |
+
"dropout": 0.1,
|
| 14 |
+
"tie_embeddings": true,
|
| 15 |
+
"pad_id": 0,
|
| 16 |
+
"max_chars": 500,
|
| 17 |
+
"answer_ids": [
|
| 18 |
+
19,
|
| 19 |
+
20
|
| 20 |
+
],
|
| 21 |
+
"special_tokens": [
|
| 22 |
+
"[PAD]",
|
| 23 |
+
"[BOS]",
|
| 24 |
+
"[USER]",
|
| 25 |
+
"[ASSISTANT]"
|
| 26 |
+
],
|
| 27 |
+
"threshold": 0.5
|
| 28 |
+
}
|
model.safetensors
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:e43803a73dd3e9bdf4e510fd33af5a07e87fb5cfef3dc38d62cc383c57e85948
|
| 3 |
+
size 4463808
|