File size: 15,880 Bytes
5c92085
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
bca5542
5c92085
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1577408
 
 
 
 
 
 
 
5c92085
1577408
 
 
 
 
 
5c92085
 
 
1577408
 
 
 
 
 
5c92085
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
bca5542
 
 
 
 
5c92085
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2d70287
 
 
 
5c92085
 
2d70287
 
0b76257
 
 
 
 
 
2d70287
5c92085
2d70287
 
5c92085
2d70287
 
 
 
5c92085
bca5542
0b76257
 
 
5c92085
0b76257
bca5542
5c92085
 
 
 
 
 
2d70287
5c92085
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
3c569ac
 
5c92085
 
 
 
 
 
 
 
 
 
 
a7d1398
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
bca5542
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2d70287
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
a7d1398
 
 
 
2d70287
 
 
 
a7d1398
bca5542
a7d1398
 
 
 
 
 
 
 
 
 
 
 
 
2d70287
 
a7d1398
 
 
 
 
 
 
 
 
 
 
 
bca5542
 
 
 
 
2d70287
 
 
 
 
 
a7d1398
 
2d70287
a7d1398
 
 
 
 
 
 
 
 
 
2d70287
 
 
 
 
 
 
 
 
a7d1398
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
"""
Structured policy rules β€” the PolicyLM-native "flag / don't flag / exception" model.

PolicyLM's `policylm_infer.Category` takes three separate fields:

    Category(name,
             violation_rule="flag this",
             not_violation_rule="don't flag this",
             exception_override="exception / carve-out")

The app lets users author a policy as a list of such rules. The SAME rules are:
  * rendered to a plain-text policy (`render_rules_to_text`) that every text model
    (gpt-oss-safeguard, Shieldstral, CoPE) receives via the normal `policy` string,
    so both model families are judged on an identical policy; and
  * sent natively to PolicyLM as discrete categories (`rules_to_categories`), so
    exceptions are read as exceptions and each rule gets its own score.

A "rule" is a plain dict: {"name", "flag", "dont_flag", "exception"} (all strings).
"""

from __future__ import annotations

import math

from utils.constants import POLICYLM_MAX_CATEGORIES, POLICYLM_MAX_POLICY_TOKENS

# The empty-rule template. Copy it (`dict(EMPTY_RULE)`) β€” never mutate the shared one.
EMPTY_RULE = {"name": "", "flag": "", "dont_flag": "", "exception": "", "enabled": True, "context": ""}

_FIELD_ORDER = ("name", "flag", "dont_flag", "exception")


def new_rule() -> dict:
    """A fresh, blank rule dict."""
    return dict(EMPTY_RULE)


def _s(rule: dict, field: str) -> str:
    """Safely read and strip a rule field."""
    return (rule.get(field) or "").strip()


def rule_has_content(rule: dict) -> bool:
    """True if the rule has anything worth sending (a name or any rule text)."""
    return any(_s(rule, f) for f in _FIELD_ORDER)


def rule_is_enabled(rule: dict) -> bool:
    """Whether this rule is switched on.

    Missing means on, so rules saved before the toggle existed keep working.
    """
    return bool(rule.get("enabled", True))


def rules_have_content(rules: list[dict] | None) -> bool:
    """True if at least one enabled rule carries content.

    Switching every rule off is a way of saying "use the free-form policy instead",
    so it must read the same as having no rules at all.
    """
    return any(rule_has_content(r) and rule_is_enabled(r) for r in (rules or []))


def nonempty_rules(rules: list[dict] | None) -> list[dict]:
    """Only the rules that carry content and are switched on.

    Every consumer funnels through here β€” the text rendering and the PolicyLM
    category payload both β€” so the toggle is honoured in one place.
    """
    return [r for r in (rules or []) if rule_has_content(r) and rule_is_enabled(r)]


def render_rules_to_text(rules: list[dict] | None) -> str:
    """
    Render structured rules into a plain-text policy document.

    This is what the text models see. It is intentionally literal about the three
    PolicyLM fields so a reading model treats "Do not flag" / "Exception" as
    carve-outs rather than things to flag.
    """
    kept = nonempty_rules(rules)
    if not kept:
        return ""

    lines = [
        "# Content Policy",
        "",
        "Apply the following rules. Content matching a rule's **Flag this** description "
        "is a violation, UNLESS it matches that rule's **Do not flag** description or an "
        "**Exception**. Evaluate each rule independently.",
        "",
    ]
    seen_context = list(dict.fromkeys(_s(r, "context") for r in kept if _s(r, "context")))
    if seen_context:
        # Once at the top for models that read the whole document; the per-rule copies
        # exist only for models scored a category at a time.
        lines += ["**Definitions and scope:** " + " ".join(seen_context), ""]
    for i, rule in enumerate(kept, start=1):
        name = _s(rule, "name") or f"Rule {i}"
        lines.append(f"## {i}. {name}")
        flag = _s(rule, "flag")
        dont = _s(rule, "dont_flag")
        exc = _s(rule, "exception")
        if flag:
            lines.append(f"- **Flag this (violation):** {flag}")
        if dont:
            lines.append(f"- **Do not flag:** {dont}")
        if exc:
            lines.append(f"- **Exception:** {exc}")
        lines.append("")
    return "\n".join(lines).rstrip() + "\n"


def rules_to_categories(rules: list[dict] | None) -> list[dict]:
    """
    Convert rules into the category payload the PolicyLM endpoint handler expects.

    Each category maps directly onto policylm_infer.Category's constructor kwargs.
    Only non-empty optional fields are included so the handler can omit kwargs it
    doesn't have values for. A rule with a "flag this" field is a scorable category; a
    rule with only a "don't flag" / "exception" (an "applies to all" carve-out) is
    policy-wide guidance and is folded into every real category rather than dropped,
    so PolicyLM honours it like the prompted models do.
    """
    categories = []
    global_dont: list[str] = []
    global_exc: list[str] = []
    # PolicyLM keys its results by category title and rejects the whole policy if two
    # share one, so a second rule called "Zone 2: ..." must not reach the endpoint
    # under that name. Two rules really can carry the same title β€” the same preset
    # added twice, or two hand-written rules β€” and failing the run outright is a worse
    # answer than scoring them separately, so repeats are numbered.
    seen_names: dict[str, int] = {}
    for rule in nonempty_rules(rules):
        flag = _s(rule, "flag")
        dont = _s(rule, "dont_flag")
        exc = _s(rule, "exception")
        if not flag:
            if dont:
                global_dont.append(dont)
            if exc:
                global_exc.append(exc)
            continue
        context = _s(rule, "context")
        base_name = _s(rule, "name") or f"Rule {len(categories) + 1}"
        n = seen_names.get(base_name.casefold(), 0) + 1
        seen_names[base_name.casefold()] = n
        cat = {
            "name": base_name if n == 1 else f"{base_name} ({n})",
            "violation_rule": f"{flag} ({context})" if context else flag,
        }
        if dont:
            cat["not_violation_rule"] = dont
        if exc:
            cat["exception_override"] = exc
        categories.append(cat)
    return _fold_global(categories, _one_line(" ".join(global_dont)), _one_line(" ".join(global_exc)))


def estimate_policy_tokens(rules: list[dict] | None) -> int:
    """
    Rough token estimate for PolicyLM's 1,662 *policy-token* budget.

    PolicyLM uses its own tokenizer; we don't load it here (keeps the Space light),
    so this is a deliberately conservative heuristic β€” max of a chars/4 and a
    words*4/3 estimate β€” that tends to warn slightly *early* rather than late. Only
    the rule text that becomes category content is counted (not the UI scaffolding).
    """
    text_parts: list[str] = []
    for rule in nonempty_rules(rules):
        for f in _FIELD_ORDER:
            v = _s(rule, f)
            if v:
                text_parts.append(v)
    text = " ".join(text_parts)
    if not text:
        return 0
    char_est = len(text) / 4
    word_est = len(text.split()) * 4 / 3
    return int(math.ceil(max(char_est, word_est)))


def policy_limits_markdown(rules: list[dict] | None) -> str:
    """
    A compact live status line for the editor: rule count, estimated tokens, and
    warnings when either PolicyLM limit is exceeded.
    """
    n = len(nonempty_rules(rules))
    tokens = estimate_policy_tokens(rules)

    cat_ok = n <= POLICYLM_MAX_CATEGORIES
    tok_ok = tokens <= POLICYLM_MAX_POLICY_TOKENS

    cat_mark = "" if cat_ok else " ⚠️"
    tok_mark = "" if tok_ok else " ⚠️"
    lines = [
        f"**Rules:** {n} / {POLICYLM_MAX_CATEGORIES}{cat_mark}  Β·  "
        f"**Est. policy tokens:** ~{tokens:,} / {POLICYLM_MAX_POLICY_TOKENS:,}{tok_mark}",
    ]

    if not cat_ok:
        lines.append(
            f"> ⚠️ **Too many rules for PolicyLM.** It supports at most "
            f"{POLICYLM_MAX_CATEGORIES} categories (a schema-model limit; prompted "
            f"models are unaffected); extra rules will be rejected by the "
            f"endpoint. The text models (gpt-oss-safeguard, Shieldstral, CoPE) still "
            f"accept all of them."
        )
    if not tok_ok:
        lines.append(
            f"> ⚠️ **Policy likely too long for PolicyLM** (est. ~{tokens:,} tokens > "
            f"{POLICYLM_MAX_POLICY_TOKENS:,}). Shorten the rule text or split the policy; "
            f"PolicyLM may truncate or error. Estimate is approximate β€” the real limit "
            f"uses PolicyLM's own tokenizer."
        )
    return "\n\n".join(lines)


# --- Free-form policy -> PolicyLM categories ---------------------------------------
#
# PolicyLM requires every category field to be a single line. A written policy is a
# multi-line markdown document, so sending one as a catch-all category fails with
# "Violation rule contains a line break; each field is one line" β€” which is what the
# legacy fallback in policylm_handler.py does.
#
# The workshop policies are already written in the shape PolicyLM wants: a "## Heading"
# per rule with "Flag:", "Don't flag:" and "Exception:" lines under it. Parsing that
# into real categories fixes the error and is strictly better than a catch-all, because
# each rule gets its own score and exceptions are read as exceptions.

_FIELD_PREFIXES = (
    ("violation_rule", ("flag:",)),
    ("not_violation_rule", ("don't flag:", "dont flag:", "do not flag:")),
    ("exception_override", ("exception:", "exceptions:")),
)


def _one_line(text: str) -> str:
    """Collapse to a single line; PolicyLM rejects any field containing a newline."""
    return " ".join(str(text).split())


def policy_definitions(policy_text: str) -> list[tuple[str, str]]:
    """Definitions stated before the first "## " heading, as (term, line) pairs.

    A policy states its scope once at the top β€” "Protected classes: …",
    "Not protected: …". A model handed the whole document reads that; a model scored
    one category at a time never sees it, which measurably changes verdicts.

    Only lines of the form "Term: definition" count. Prose in the preamble is left for
    the rendered document, since there is no term to attach it to.
    """
    head = (policy_text or "").split("\n## ")[0]
    out: list[tuple[str, str]] = []
    for raw in head.splitlines():
        line = raw.strip().lstrip("-* ").strip()
        if line.startswith("#") or ":" not in line:
            continue
        term, _, rest = line.partition(":")
        if term and rest.strip() and len(term.split()) <= 4:
            out.append((_one_line(term).lower(), _one_line(line)))
    return out


def _relevant_definitions(text: str, definitions: list[tuple[str, str]]) -> str:
    """The definitions this rule actually relies on.

    Attached per rule rather than to all of them: repeating every definition on every
    category burns the token budget and tells a model about terms its rule never uses.
    A term matches on its longest word ("protected" for "Protected classes"), which
    also catches the singular.
    """
    lowered = (text or "").lower()
    kept = [
        line for term, line in definitions
        if any(w for w in sorted(term.split(), key=len, reverse=True)[:1] if w in lowered)
    ]
    return " ".join(dict.fromkeys(kept))


def _fold_global(categories: list[dict], global_dont: str, global_exc: str) -> list[dict]:
    """Merge policy-wide carve-outs into every category.

    PolicyLM scores each category independently, so a "## Applies to all categories"
    section β€” a Don't-flag / Exception clause with no Flag of its own β€” only takes
    effect if it is present in each category. Without this it is dropped (nothing to
    score), so PolicyLM flags content the carve-out protects while the prompted models,
    reading the whole document, do not.
    """
    for cat in categories:
        if global_dont:
            cat["not_violation_rule"] = " ".join(
                p for p in (cat.get("not_violation_rule", ""), global_dont) if p
            )
        if global_exc:
            cat["exception_override"] = " ".join(
                p for p in (cat.get("exception_override", ""), global_exc) if p
            )
    return categories


def parse_policy_markdown(policy_text: str, max_categories: int = POLICYLM_MAX_CATEGORIES) -> list[dict]:
    """Parse a written policy into PolicyLM categories.

    Recognises the "## Name" + "Flag:/Don't flag:/Exception:" shape the workshop
    policies use. A section with no Flag line is not its own category (nothing to
    score) but, if it carries a Don't-flag / Exception clause (an "applies to all"
    carve-out), that clause is folded into every real category rather than dropped.
    Returns an empty list when the document has no parseable sections.
    """
    definitions = policy_definitions(policy_text)
    sections: list[tuple[str, list[str]]] = []
    current: tuple[str, list[str]] | None = None
    for line in (policy_text or "").splitlines():
        if line.startswith("## "):
            if current:
                sections.append(current)
            current = (line[3:].strip(), [])
        elif current:
            current[1].append(line)
    if current:
        sections.append(current)

    categories: list[dict] = []
    global_dont: list[str] = []
    global_exc: list[str] = []
    for name, body in sections:
        found: dict[str, str] = {}
        for raw in body:
            stripped = raw.strip().lstrip("-* ").strip()
            lowered = stripped.lower()
            for field, prefixes in _FIELD_PREFIXES:
                for prefix in prefixes:
                    if lowered.startswith(prefix) and field not in found:
                        value = _one_line(stripped[len(prefix):])
                        if value:
                            found[field] = value
        if "violation_rule" in found:
            entry = {"name": _one_line(name)[:120], **found}
            context = _relevant_definitions(" ".join([name, *found.values()]), definitions)
            if context:
                entry["context"] = context
            categories.append(entry)
        else:
            # Flag-less section = policy-wide guidance; keep its carve-out clauses.
            if found.get("not_violation_rule"):
                global_dont.append(found["not_violation_rule"])
            if found.get("exception_override"):
                global_exc.append(found["exception_override"])
        if len(categories) >= max_categories:
            break
    return _fold_global(categories, _one_line(" ".join(global_dont)), _one_line(" ".join(global_exc)))


def policy_text_to_categories(policy_text: str, max_categories: int = POLICYLM_MAX_CATEGORIES) -> list[dict]:
    """Categories for a free-form policy, always safe to send to PolicyLM.

    Falls back to a single flattened catch-all when the document has no recognisable
    sections β€” worse than per-rule scoring, but it runs instead of erroring.
    """
    categories = parse_policy_markdown(policy_text, max_categories)
    if categories:
        # Fold the per-category definition context into the violation rule, the same way
        # rules_to_categories does for the editor path. The endpoint handler only reads
        # name/violation_rule/not_violation_rule/exception_override and ignores a separate
        # "context" key, so on this free-form path (which bypasses the editor) the policy's
        # preamble definitions would otherwise never reach PolicyLM.
        for cat in categories:
            context = cat.pop("context", "")
            if context:
                cat["violation_rule"] = f"{cat['violation_rule']} ({context})"
        return categories
    flattened = _one_line(policy_text)
    return [{"name": "Policy violation", "violation_rule": flattened}] if flattened else []