File size: 1,439 Bytes
043267b
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
"""Zero-dependency fallback (same idea as extension/src/lib/labelset.js rulesClassify)."""
from __future__ import annotations

import math
import re
import unicodedata

from ..spec import LabelSet, Post, Result


def _tok(s: str) -> list[str]:
    s = unicodedata.normalize("NFKD", (s or "").lower())
    s = "".join(c for c in s if not unicodedata.combining(c))
    return [w for w in re.split(r"[^\w]+", s) if len(w) > 2]


class RulesBackend:
    name = "rules"

    def classify_one(self, ls: LabelSet, text: str) -> Result:
        t = set(_tok(text))
        scores: dict[str, float] = {}
        for l in ls.labels:
            bag = set(_tok(l.rubric))
            for e in ls.examples:
                if e.label == l.key:
                    bag |= set(_tok(e.text))
            inter = len(t & bag)
            scores[l.key] = (inter + 0.1) / (math.sqrt(len(t) * len(bag)) + 1)
        noise = next((l for l in ls.labels if l.action in ("collapse", "hide")), None)
        if noise and len(t) <= 2:
            scores[noise.key] += 1
        total = sum(scores.values()) or 1.0
        probs = {k: v / total for k, v in scores.items()}
        best = max(probs, key=probs.get)
        return Result(label=best, probabilities=probs, confidence=probs[best], backend=self.name)

    async def classify(self, ls: LabelSet, posts: list[Post]) -> dict[str, Result]:
        return {p.id: self.classify_one(ls, p.text) for p in posts}