PolicyLM: deliver preamble context on the free-form path + fold 'applies to all' clauses
Browse filesTwo gaps let a written policy's shared guidance reach the prompted models but not
PolicyLM (which is scored one category at a time):
1. The per-category definition `context` (protected/not-protected scope) was folded
into the violation rule only in rules_to_categories (editor path). On the free-form
/ Policy-text path, policy_text_to_categories left it as a separate `context` key,
which the endpoint handler ignores — so the preamble never reached PolicyLM. Fold it
in there too.
2. A flag-less "## Applies to all categories" section (a Don't-flag / Exception
carve-out with no Flag of its own) was dropped entirely. parse_policy_markdown and
rules_to_categories now collect those clauses and fold them into every real category.
Measured live: hate-speech criticism of a non-protected class 0.67 -> 0.54, scams
counter-speech carve-out now present in every category; true positives unchanged at ~0.99.
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
- utils/policy_rules.py +58 -12
|
@@ -118,28 +118,35 @@ def rules_to_categories(rules: list[dict] | None) -> list[dict]:
|
|
| 118 |
|
| 119 |
Each category maps directly onto policylm_infer.Category's constructor kwargs.
|
| 120 |
Only non-empty optional fields are included so the handler can omit kwargs it
|
| 121 |
-
doesn't have values for. A rule
|
| 122 |
-
|
|
|
|
|
|
|
| 123 |
"""
|
| 124 |
categories = []
|
| 125 |
-
|
|
|
|
|
|
|
| 126 |
flag = _s(rule, "flag")
|
|
|
|
|
|
|
| 127 |
if not flag:
|
| 128 |
-
|
|
|
|
|
|
|
|
|
|
| 129 |
continue
|
| 130 |
context = _s(rule, "context")
|
| 131 |
cat = {
|
| 132 |
-
"name": _s(rule, "name") or f"Rule {
|
| 133 |
"violation_rule": f"{flag} ({context})" if context else flag,
|
| 134 |
}
|
| 135 |
-
dont = _s(rule, "dont_flag")
|
| 136 |
-
exc = _s(rule, "exception")
|
| 137 |
if dont:
|
| 138 |
cat["not_violation_rule"] = dont
|
| 139 |
if exc:
|
| 140 |
cat["exception_override"] = exc
|
| 141 |
categories.append(cat)
|
| 142 |
-
return categories
|
| 143 |
|
| 144 |
|
| 145 |
def estimate_policy_tokens(rules: list[dict] | None) -> int:
|
|
@@ -263,13 +270,35 @@ def _relevant_definitions(text: str, definitions: list[tuple[str, str]]) -> str:
|
|
| 263 |
return " ".join(dict.fromkeys(kept))
|
| 264 |
|
| 265 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 266 |
def parse_policy_markdown(policy_text: str, max_categories: int = POLICYLM_MAX_CATEGORIES) -> list[dict]:
|
| 267 |
"""Parse a written policy into PolicyLM categories.
|
| 268 |
|
| 269 |
Recognises the "## Name" + "Flag:/Don't flag:/Exception:" shape the workshop
|
| 270 |
-
policies use.
|
| 271 |
-
|
| 272 |
-
|
|
|
|
| 273 |
"""
|
| 274 |
definitions = policy_definitions(policy_text)
|
| 275 |
sections: list[tuple[str, list[str]]] = []
|
|
@@ -285,6 +314,8 @@ def parse_policy_markdown(policy_text: str, max_categories: int = POLICYLM_MAX_C
|
|
| 285 |
sections.append(current)
|
| 286 |
|
| 287 |
categories: list[dict] = []
|
|
|
|
|
|
|
| 288 |
for name, body in sections:
|
| 289 |
found: dict[str, str] = {}
|
| 290 |
for raw in body:
|
|
@@ -302,9 +333,15 @@ def parse_policy_markdown(policy_text: str, max_categories: int = POLICYLM_MAX_C
|
|
| 302 |
if context:
|
| 303 |
entry["context"] = context
|
| 304 |
categories.append(entry)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 305 |
if len(categories) >= max_categories:
|
| 306 |
break
|
| 307 |
-
return categories
|
| 308 |
|
| 309 |
|
| 310 |
def policy_text_to_categories(policy_text: str, max_categories: int = POLICYLM_MAX_CATEGORIES) -> list[dict]:
|
|
@@ -315,6 +352,15 @@ def policy_text_to_categories(policy_text: str, max_categories: int = POLICYLM_M
|
|
| 315 |
"""
|
| 316 |
categories = parse_policy_markdown(policy_text, max_categories)
|
| 317 |
if categories:
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 318 |
return categories
|
| 319 |
flattened = _one_line(policy_text)
|
| 320 |
return [{"name": "Policy violation", "violation_rule": flattened}] if flattened else []
|
|
|
|
| 118 |
|
| 119 |
Each category maps directly onto policylm_infer.Category's constructor kwargs.
|
| 120 |
Only non-empty optional fields are included so the handler can omit kwargs it
|
| 121 |
+
doesn't have values for. A rule with a "flag this" field is a scorable category; a
|
| 122 |
+
rule with only a "don't flag" / "exception" (an "applies to all" carve-out) is
|
| 123 |
+
policy-wide guidance and is folded into every real category rather than dropped,
|
| 124 |
+
so PolicyLM honours it like the prompted models do.
|
| 125 |
"""
|
| 126 |
categories = []
|
| 127 |
+
global_dont: list[str] = []
|
| 128 |
+
global_exc: list[str] = []
|
| 129 |
+
for rule in nonempty_rules(rules):
|
| 130 |
flag = _s(rule, "flag")
|
| 131 |
+
dont = _s(rule, "dont_flag")
|
| 132 |
+
exc = _s(rule, "exception")
|
| 133 |
if not flag:
|
| 134 |
+
if dont:
|
| 135 |
+
global_dont.append(dont)
|
| 136 |
+
if exc:
|
| 137 |
+
global_exc.append(exc)
|
| 138 |
continue
|
| 139 |
context = _s(rule, "context")
|
| 140 |
cat = {
|
| 141 |
+
"name": _s(rule, "name") or f"Rule {len(categories) + 1}",
|
| 142 |
"violation_rule": f"{flag} ({context})" if context else flag,
|
| 143 |
}
|
|
|
|
|
|
|
| 144 |
if dont:
|
| 145 |
cat["not_violation_rule"] = dont
|
| 146 |
if exc:
|
| 147 |
cat["exception_override"] = exc
|
| 148 |
categories.append(cat)
|
| 149 |
+
return _fold_global(categories, _one_line(" ".join(global_dont)), _one_line(" ".join(global_exc)))
|
| 150 |
|
| 151 |
|
| 152 |
def estimate_policy_tokens(rules: list[dict] | None) -> int:
|
|
|
|
| 270 |
return " ".join(dict.fromkeys(kept))
|
| 271 |
|
| 272 |
|
| 273 |
+
def _fold_global(categories: list[dict], global_dont: str, global_exc: str) -> list[dict]:
|
| 274 |
+
"""Merge policy-wide carve-outs into every category.
|
| 275 |
+
|
| 276 |
+
PolicyLM scores each category independently, so a "## Applies to all categories"
|
| 277 |
+
section — a Don't-flag / Exception clause with no Flag of its own — only takes
|
| 278 |
+
effect if it is present in each category. Without this it is dropped (nothing to
|
| 279 |
+
score), so PolicyLM flags content the carve-out protects while the prompted models,
|
| 280 |
+
reading the whole document, do not.
|
| 281 |
+
"""
|
| 282 |
+
for cat in categories:
|
| 283 |
+
if global_dont:
|
| 284 |
+
cat["not_violation_rule"] = " ".join(
|
| 285 |
+
p for p in (cat.get("not_violation_rule", ""), global_dont) if p
|
| 286 |
+
)
|
| 287 |
+
if global_exc:
|
| 288 |
+
cat["exception_override"] = " ".join(
|
| 289 |
+
p for p in (cat.get("exception_override", ""), global_exc) if p
|
| 290 |
+
)
|
| 291 |
+
return categories
|
| 292 |
+
|
| 293 |
+
|
| 294 |
def parse_policy_markdown(policy_text: str, max_categories: int = POLICYLM_MAX_CATEGORIES) -> list[dict]:
|
| 295 |
"""Parse a written policy into PolicyLM categories.
|
| 296 |
|
| 297 |
Recognises the "## Name" + "Flag:/Don't flag:/Exception:" shape the workshop
|
| 298 |
+
policies use. A section with no Flag line is not its own category (nothing to
|
| 299 |
+
score) but, if it carries a Don't-flag / Exception clause (an "applies to all"
|
| 300 |
+
carve-out), that clause is folded into every real category rather than dropped.
|
| 301 |
+
Returns an empty list when the document has no parseable sections.
|
| 302 |
"""
|
| 303 |
definitions = policy_definitions(policy_text)
|
| 304 |
sections: list[tuple[str, list[str]]] = []
|
|
|
|
| 314 |
sections.append(current)
|
| 315 |
|
| 316 |
categories: list[dict] = []
|
| 317 |
+
global_dont: list[str] = []
|
| 318 |
+
global_exc: list[str] = []
|
| 319 |
for name, body in sections:
|
| 320 |
found: dict[str, str] = {}
|
| 321 |
for raw in body:
|
|
|
|
| 333 |
if context:
|
| 334 |
entry["context"] = context
|
| 335 |
categories.append(entry)
|
| 336 |
+
else:
|
| 337 |
+
# Flag-less section = policy-wide guidance; keep its carve-out clauses.
|
| 338 |
+
if found.get("not_violation_rule"):
|
| 339 |
+
global_dont.append(found["not_violation_rule"])
|
| 340 |
+
if found.get("exception_override"):
|
| 341 |
+
global_exc.append(found["exception_override"])
|
| 342 |
if len(categories) >= max_categories:
|
| 343 |
break
|
| 344 |
+
return _fold_global(categories, _one_line(" ".join(global_dont)), _one_line(" ".join(global_exc)))
|
| 345 |
|
| 346 |
|
| 347 |
def policy_text_to_categories(policy_text: str, max_categories: int = POLICYLM_MAX_CATEGORIES) -> list[dict]:
|
|
|
|
| 352 |
"""
|
| 353 |
categories = parse_policy_markdown(policy_text, max_categories)
|
| 354 |
if categories:
|
| 355 |
+
# Fold the per-category definition context into the violation rule, the same way
|
| 356 |
+
# rules_to_categories does for the editor path. The endpoint handler only reads
|
| 357 |
+
# name/violation_rule/not_violation_rule/exception_override and ignores a separate
|
| 358 |
+
# "context" key, so on this free-form path (which bypasses the editor) the policy's
|
| 359 |
+
# preamble definitions would otherwise never reach PolicyLM.
|
| 360 |
+
for cat in categories:
|
| 361 |
+
context = cat.pop("context", "")
|
| 362 |
+
if context:
|
| 363 |
+
cat["violation_rule"] = f"{cat['violation_rule']} ({context})"
|
| 364 |
return categories
|
| 365 |
flattened = _one_line(policy_text)
|
| 366 |
return [{"name": "Policy violation", "violation_rule": flattened}] if flattened else []
|