""" Structured policy rules — the PolicyLM-native "flag / don't flag / exception" model. PolicyLM's `policylm_infer.Category` takes three separate fields: Category(name, violation_rule="flag this", not_violation_rule="don't flag this", exception_override="exception / carve-out") The app lets users author a policy as a list of such rules. The SAME rules are: * rendered to a plain-text policy (`render_rules_to_text`) that every text model (gpt-oss-safeguard, Shieldstral, CoPE) receives via the normal `policy` string, so both model families are judged on an identical policy; and * sent natively to PolicyLM as discrete categories (`rules_to_categories`), so exceptions are read as exceptions and each rule gets its own score. A "rule" is a plain dict: {"name", "flag", "dont_flag", "exception"} (all strings). """ from __future__ import annotations import math from utils.constants import POLICYLM_MAX_CATEGORIES, POLICYLM_MAX_POLICY_TOKENS # The empty-rule template. Copy it (`dict(EMPTY_RULE)`) — never mutate the shared one. EMPTY_RULE = {"name": "", "flag": "", "dont_flag": "", "exception": "", "enabled": True, "context": ""} _FIELD_ORDER = ("name", "flag", "dont_flag", "exception") def new_rule() -> dict: """A fresh, blank rule dict.""" return dict(EMPTY_RULE) def _s(rule: dict, field: str) -> str: """Safely read and strip a rule field.""" return (rule.get(field) or "").strip() def rule_has_content(rule: dict) -> bool: """True if the rule has anything worth sending (a name or any rule text).""" return any(_s(rule, f) for f in _FIELD_ORDER) def rule_is_enabled(rule: dict) -> bool: """Whether this rule is switched on. Missing means on, so rules saved before the toggle existed keep working. """ return bool(rule.get("enabled", True)) def rules_have_content(rules: list[dict] | None) -> bool: """True if at least one enabled rule carries content. Switching every rule off is a way of saying "use the free-form policy instead", so it must read the same as having no rules at all. """ return any(rule_has_content(r) and rule_is_enabled(r) for r in (rules or [])) def nonempty_rules(rules: list[dict] | None) -> list[dict]: """Only the rules that carry content and are switched on. Every consumer funnels through here — the text rendering and the PolicyLM category payload both — so the toggle is honoured in one place. """ return [r for r in (rules or []) if rule_has_content(r) and rule_is_enabled(r)] def render_rules_to_text(rules: list[dict] | None) -> str: """ Render structured rules into a plain-text policy document. This is what the text models see. It is intentionally literal about the three PolicyLM fields so a reading model treats "Do not flag" / "Exception" as carve-outs rather than things to flag. """ kept = nonempty_rules(rules) if not kept: return "" lines = [ "# Content Policy", "", "Apply the following rules. Content matching a rule's **Flag this** description " "is a violation, UNLESS it matches that rule's **Do not flag** description or an " "**Exception**. Evaluate each rule independently.", "", ] seen_context = list(dict.fromkeys(_s(r, "context") for r in kept if _s(r, "context"))) if seen_context: # Once at the top for models that read the whole document; the per-rule copies # exist only for models scored a category at a time. lines += ["**Definitions and scope:** " + " ".join(seen_context), ""] for i, rule in enumerate(kept, start=1): name = _s(rule, "name") or f"Rule {i}" lines.append(f"## {i}. {name}") flag = _s(rule, "flag") dont = _s(rule, "dont_flag") exc = _s(rule, "exception") if flag: lines.append(f"- **Flag this (violation):** {flag}") if dont: lines.append(f"- **Do not flag:** {dont}") if exc: lines.append(f"- **Exception:** {exc}") lines.append("") return "\n".join(lines).rstrip() + "\n" def rules_to_categories(rules: list[dict] | None) -> list[dict]: """ Convert rules into the category payload the PolicyLM endpoint handler expects. Each category maps directly onto policylm_infer.Category's constructor kwargs. Only non-empty optional fields are included so the handler can omit kwargs it doesn't have values for. A rule with a "flag this" field is a scorable category; a rule with only a "don't flag" / "exception" (an "applies to all" carve-out) is policy-wide guidance and is folded into every real category rather than dropped, so PolicyLM honours it like the prompted models do. """ categories = [] global_dont: list[str] = [] global_exc: list[str] = [] # PolicyLM keys its results by category title and rejects the whole policy if two # share one, so a second rule called "Zone 2: ..." must not reach the endpoint # under that name. Two rules really can carry the same title — the same preset # added twice, or two hand-written rules — and failing the run outright is a worse # answer than scoring them separately, so repeats are numbered. seen_names: dict[str, int] = {} for rule in nonempty_rules(rules): flag = _s(rule, "flag") dont = _s(rule, "dont_flag") exc = _s(rule, "exception") if not flag: if dont: global_dont.append(dont) if exc: global_exc.append(exc) continue context = _s(rule, "context") base_name = _s(rule, "name") or f"Rule {len(categories) + 1}" n = seen_names.get(base_name.casefold(), 0) + 1 seen_names[base_name.casefold()] = n cat = { "name": base_name if n == 1 else f"{base_name} ({n})", "violation_rule": f"{flag} ({context})" if context else flag, } if dont: cat["not_violation_rule"] = dont if exc: cat["exception_override"] = exc categories.append(cat) return _fold_global(categories, _one_line(" ".join(global_dont)), _one_line(" ".join(global_exc))) def estimate_policy_tokens(rules: list[dict] | None) -> int: """ Rough token estimate for PolicyLM's 1,662 *policy-token* budget. PolicyLM uses its own tokenizer; we don't load it here (keeps the Space light), so this is a deliberately conservative heuristic — max of a chars/4 and a words*4/3 estimate — that tends to warn slightly *early* rather than late. Only the rule text that becomes category content is counted (not the UI scaffolding). """ text_parts: list[str] = [] for rule in nonempty_rules(rules): for f in _FIELD_ORDER: v = _s(rule, f) if v: text_parts.append(v) text = " ".join(text_parts) if not text: return 0 char_est = len(text) / 4 word_est = len(text.split()) * 4 / 3 return int(math.ceil(max(char_est, word_est))) def policy_limits_markdown(rules: list[dict] | None) -> str: """ A compact live status line for the editor: rule count, estimated tokens, and warnings when either PolicyLM limit is exceeded. """ n = len(nonempty_rules(rules)) tokens = estimate_policy_tokens(rules) cat_ok = n <= POLICYLM_MAX_CATEGORIES tok_ok = tokens <= POLICYLM_MAX_POLICY_TOKENS cat_mark = "" if cat_ok else " ⚠️" tok_mark = "" if tok_ok else " ⚠️" lines = [ f"**Rules:** {n} / {POLICYLM_MAX_CATEGORIES}{cat_mark} · " f"**Est. policy tokens:** ~{tokens:,} / {POLICYLM_MAX_POLICY_TOKENS:,}{tok_mark}", ] if not cat_ok: lines.append( f"> ⚠️ **Too many rules for PolicyLM.** It supports at most " f"{POLICYLM_MAX_CATEGORIES} categories (a schema-model limit; prompted " f"models are unaffected); extra rules will be rejected by the " f"endpoint. The text models (gpt-oss-safeguard, Shieldstral, CoPE) still " f"accept all of them." ) if not tok_ok: lines.append( f"> ⚠️ **Policy likely too long for PolicyLM** (est. ~{tokens:,} tokens > " f"{POLICYLM_MAX_POLICY_TOKENS:,}). Shorten the rule text or split the policy; " f"PolicyLM may truncate or error. Estimate is approximate — the real limit " f"uses PolicyLM's own tokenizer." ) return "\n\n".join(lines) # --- Free-form policy -> PolicyLM categories --------------------------------------- # # PolicyLM requires every category field to be a single line. A written policy is a # multi-line markdown document, so sending one as a catch-all category fails with # "Violation rule contains a line break; each field is one line" — which is what the # legacy fallback in policylm_handler.py does. # # The workshop policies are already written in the shape PolicyLM wants: a "## Heading" # per rule with "Flag:", "Don't flag:" and "Exception:" lines under it. Parsing that # into real categories fixes the error and is strictly better than a catch-all, because # each rule gets its own score and exceptions are read as exceptions. _FIELD_PREFIXES = ( ("violation_rule", ("flag:",)), ("not_violation_rule", ("don't flag:", "dont flag:", "do not flag:")), ("exception_override", ("exception:", "exceptions:")), ) def _one_line(text: str) -> str: """Collapse to a single line; PolicyLM rejects any field containing a newline.""" return " ".join(str(text).split()) def policy_definitions(policy_text: str) -> list[tuple[str, str]]: """Definitions stated before the first "## " heading, as (term, line) pairs. A policy states its scope once at the top — "Protected classes: …", "Not protected: …". A model handed the whole document reads that; a model scored one category at a time never sees it, which measurably changes verdicts. Only lines of the form "Term: definition" count. Prose in the preamble is left for the rendered document, since there is no term to attach it to. """ head = (policy_text or "").split("\n## ")[0] out: list[tuple[str, str]] = [] for raw in head.splitlines(): line = raw.strip().lstrip("-* ").strip() if line.startswith("#") or ":" not in line: continue term, _, rest = line.partition(":") if term and rest.strip() and len(term.split()) <= 4: out.append((_one_line(term).lower(), _one_line(line))) return out def _relevant_definitions(text: str, definitions: list[tuple[str, str]]) -> str: """The definitions this rule actually relies on. Attached per rule rather than to all of them: repeating every definition on every category burns the token budget and tells a model about terms its rule never uses. A term matches on its longest word ("protected" for "Protected classes"), which also catches the singular. """ lowered = (text or "").lower() kept = [ line for term, line in definitions if any(w for w in sorted(term.split(), key=len, reverse=True)[:1] if w in lowered) ] return " ".join(dict.fromkeys(kept)) def _fold_global(categories: list[dict], global_dont: str, global_exc: str) -> list[dict]: """Merge policy-wide carve-outs into every category. PolicyLM scores each category independently, so a "## Applies to all categories" section — a Don't-flag / Exception clause with no Flag of its own — only takes effect if it is present in each category. Without this it is dropped (nothing to score), so PolicyLM flags content the carve-out protects while the prompted models, reading the whole document, do not. """ for cat in categories: if global_dont: cat["not_violation_rule"] = " ".join( p for p in (cat.get("not_violation_rule", ""), global_dont) if p ) if global_exc: cat["exception_override"] = " ".join( p for p in (cat.get("exception_override", ""), global_exc) if p ) return categories def parse_policy_markdown(policy_text: str, max_categories: int = POLICYLM_MAX_CATEGORIES) -> list[dict]: """Parse a written policy into PolicyLM categories. Recognises the "## Name" + "Flag:/Don't flag:/Exception:" shape the workshop policies use. A section with no Flag line is not its own category (nothing to score) but, if it carries a Don't-flag / Exception clause (an "applies to all" carve-out), that clause is folded into every real category rather than dropped. Returns an empty list when the document has no parseable sections. """ definitions = policy_definitions(policy_text) sections: list[tuple[str, list[str]]] = [] current: tuple[str, list[str]] | None = None for line in (policy_text or "").splitlines(): if line.startswith("## "): if current: sections.append(current) current = (line[3:].strip(), []) elif current: current[1].append(line) if current: sections.append(current) categories: list[dict] = [] global_dont: list[str] = [] global_exc: list[str] = [] for name, body in sections: found: dict[str, str] = {} for raw in body: stripped = raw.strip().lstrip("-* ").strip() lowered = stripped.lower() for field, prefixes in _FIELD_PREFIXES: for prefix in prefixes: if lowered.startswith(prefix) and field not in found: value = _one_line(stripped[len(prefix):]) if value: found[field] = value if "violation_rule" in found: entry = {"name": _one_line(name)[:120], **found} context = _relevant_definitions(" ".join([name, *found.values()]), definitions) if context: entry["context"] = context categories.append(entry) else: # Flag-less section = policy-wide guidance; keep its carve-out clauses. if found.get("not_violation_rule"): global_dont.append(found["not_violation_rule"]) if found.get("exception_override"): global_exc.append(found["exception_override"]) if len(categories) >= max_categories: break return _fold_global(categories, _one_line(" ".join(global_dont)), _one_line(" ".join(global_exc))) def policy_text_to_categories(policy_text: str, max_categories: int = POLICYLM_MAX_CATEGORIES) -> list[dict]: """Categories for a free-form policy, always safe to send to PolicyLM. Falls back to a single flattened catch-all when the document has no recognisable sections — worse than per-rule scoring, but it runs instead of erroring. """ categories = parse_policy_markdown(policy_text, max_categories) if categories: # Fold the per-category definition context into the violation rule, the same way # rules_to_categories does for the editor path. The endpoint handler only reads # name/violation_rule/not_violation_rule/exception_override and ignores a separate # "context" key, so on this free-form path (which bypasses the editor) the policy's # preamble definitions would otherwise never reach PolicyLM. for cat in categories: context = cat.pop("context", "") if context: cat["violation_rule"] = f"{cat['violation_rule']} ({context})" return categories flattened = _one_line(policy_text) return [{"name": "Policy violation", "violation_rule": flattened}] if flattened else []