SRK-CobuildX's picture
Release v0.1
d770b7e verified
Raw History Blame Contribute Delete
5.24 kB
# Copyright 2026 Cobuild Tech Private Limited
# SPDX-License-Identifier: Apache-2.0
"""CBX Indic profanity classifier: a comment or message in, four offensive-content labels out.
from cbx_profanity import ProfanityClassifier
clf = ProfanityClassifier(".") # folder with the model files
clf.classify("tu ekdum bewakoof hai")
`classify` takes one text (returns one result) or a list (returns a list). A result:
{"labels": ["insult"], "offensive": True,
"scores": {"profanity": 0.08, "insult": 0.91, "hate": 0.02, "sexual": 0.01, "offensive": 0.93},
"truncated": False}
"labels" are the labels whose score reaches its threshold (profanity_config.json). "offensive" comes from a fifth,
auxiliary head trained on "any of the four"; it is the most reliable single flag. The model reads at most 256 tokens;
longer text is cut ("truncated": True). For long documents use `classify_document`, which scores 256-token chunks
and keeps the highest score per label.
"""
import json
from pathlib import Path
import torch
import torch.nn as nn
from safetensors.torch import load_file
from transformers import AutoConfig, AutoModel, AutoTokenizer
class _Net(nn.Module):
def __init__(self, path, cfg):
super().__init__()
self.backbone = AutoModel.from_config(AutoConfig.from_pretrained(path), attn_implementation="sdpa")
h = self.backbone.config.hidden_size
if cfg.get("embedding_rank"): # factorised word table (smallest model)
self.backbone.embeddings.tok_embeddings = nn.Sequential(
nn.Embedding(self.backbone.config.vocab_size, cfg["embedding_rank"]),
nn.Linear(cfg["embedding_rank"], h, bias=False))
self.head = nn.Linear(h, len(cfg["heads"]))
def forward(self, input_ids, attention_mask):
hs = self.backbone(input_ids=input_ids, attention_mask=attention_mask).last_hidden_state.float()
m = attention_mask.unsqueeze(-1).float()
return self.head((hs[:, 0] + (hs * m).sum(1) / m.sum(1).clamp(min=1)) / 2)
class ProfanityClassifier:
def __init__(self, path=".", device=None, thresholds=None):
path = Path(path)
self.cfg = json.loads((path / "profanity_config.json").read_text())
self.thresholds = {**self.cfg["thresholds"], **(thresholds or {})}
self.labels = self.cfg["labels"]
self.heads = self.cfg["heads"]
self.max_len = self.cfg["max_len"]
self.device = device or ("cuda" if torch.cuda.is_available() else "cpu")
self.tok = AutoTokenizer.from_pretrained(path)
self.net = _Net(path, self.cfg)
self.net.load_state_dict(load_file(str(path / "model.safetensors")))
self.net.to(self.device).eval()
@torch.inference_mode()
def scores(self, texts, batch_size=64):
"""Raw sigmoid scores [n][5] (heads in profanity_config.json order) and a truncated flag per text."""
full = self.tok(list(texts), verbose=False)["input_ids"]
enc = [e[:self.max_len - 1] + [e[-1]] if len(e) > self.max_len else e for e in full] # keep the end token
out = [None] * len(enc)
order = sorted(range(len(enc)), key=lambda i: len(enc[i]))
for i in range(0, len(order), batch_size):
idx = order[i:i + batch_size]
n = max(len(enc[j]) for j in idx)
ids = torch.full((len(idx), n), self.tok.pad_token_id, dtype=torch.long)
mask = torch.zeros((len(idx), n), dtype=torch.long)
for k, j in enumerate(idx):
ids[k, :len(enc[j])] = torch.tensor(enc[j])
mask[k, :len(enc[j])] = 1
p = torch.sigmoid(self.net(ids.to(self.device), mask.to(self.device))).cpu().tolist()
for k, j in enumerate(idx):
out[j] = p[k]
return out, [len(f) > self.max_len for f in full]
def _result(self, p, truncated):
s = dict(zip(self.heads, p))
return {"labels": [l for l in self.labels if s[l] >= self.thresholds[l]],
"offensive": s["offensive"] >= self.thresholds["offensive"],
"scores": {h: round(v, 4) for h, v in s.items()}, "truncated": truncated}
def classify(self, texts, batch_size=64):
single = isinstance(texts, str)
texts = [texts] if single else list(texts)
p, tr = self.scores(texts, batch_size)
out = [self._result(a, b) for a, b in zip(p, tr)]
return out[0] if single else out
def classify_document(self, text, batch_size=64):
"""Long text: split into chunks of up to 256 tokens, score each, keep the highest score per label."""
ids = self.tok(text, add_special_tokens=False, verbose=False)["input_ids"]
step = self.max_len - 2
chunks = [self.tok.decode(ids[i:i + step]) for i in range(0, max(len(ids), 1), step)] or [text]
p, _ = self.scores(chunks, batch_size)
best = [max(c[k] for c in p) for k in range(len(self.heads))]
r = self._result(best, False)
r["chunks"] = len(chunks)
r["flagged_chunks"] = sum(any(c[k] >= self.thresholds[h] for k, h in enumerate(self.heads)
if h in self.labels) for c in p)
return r