juliets Claude Opus 4.8 (1M context) commited on
Commit
2d70287
·
1 Parent(s): bca5542

PolicyLM: deliver preamble context on the free-form path + fold 'applies to all' clauses

Browse files

Two gaps let a written policy's shared guidance reach the prompted models but not
PolicyLM (which is scored one category at a time):

1. The per-category definition `context` (protected/not-protected scope) was folded
into the violation rule only in rules_to_categories (editor path). On the free-form
/ Policy-text path, policy_text_to_categories left it as a separate `context` key,
which the endpoint handler ignores — so the preamble never reached PolicyLM. Fold it
in there too.
2. A flag-less "## Applies to all categories" section (a Don't-flag / Exception
carve-out with no Flag of its own) was dropped entirely. parse_policy_markdown and
rules_to_categories now collect those clauses and fold them into every real category.

Measured live: hate-speech criticism of a non-protected class 0.67 -> 0.54, scams
counter-speech carve-out now present in every category; true positives unchanged at ~0.99.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

Files changed (1) hide show
  1. utils/policy_rules.py +58 -12
utils/policy_rules.py CHANGED
@@ -118,28 +118,35 @@ def rules_to_categories(rules: list[dict] | None) -> list[dict]:
118
 
119
  Each category maps directly onto policylm_infer.Category's constructor kwargs.
120
  Only non-empty optional fields are included so the handler can omit kwargs it
121
- doesn't have values for. A rule needs a violation_rule (the "flag this" field)
122
- to be a meaningful category, so rules with no "flag" text are skipped.
 
 
123
  """
124
  categories = []
125
- for i, rule in enumerate(nonempty_rules(rules), start=1):
 
 
126
  flag = _s(rule, "flag")
 
 
127
  if not flag:
128
- # Without a violation rule there is nothing for PolicyLM to score.
 
 
 
129
  continue
130
  context = _s(rule, "context")
131
  cat = {
132
- "name": _s(rule, "name") or f"Rule {i}",
133
  "violation_rule": f"{flag} ({context})" if context else flag,
134
  }
135
- dont = _s(rule, "dont_flag")
136
- exc = _s(rule, "exception")
137
  if dont:
138
  cat["not_violation_rule"] = dont
139
  if exc:
140
  cat["exception_override"] = exc
141
  categories.append(cat)
142
- return categories
143
 
144
 
145
  def estimate_policy_tokens(rules: list[dict] | None) -> int:
@@ -263,13 +270,35 @@ def _relevant_definitions(text: str, definitions: list[tuple[str, str]]) -> str:
263
  return " ".join(dict.fromkeys(kept))
264
 
265
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
266
  def parse_policy_markdown(policy_text: str, max_categories: int = POLICYLM_MAX_CATEGORIES) -> list[dict]:
267
  """Parse a written policy into PolicyLM categories.
268
 
269
  Recognises the "## Name" + "Flag:/Don't flag:/Exception:" shape the workshop
270
- policies use. Sections without a Flag line are skipped, since a category with no
271
- violation rule is nothing for PolicyLM to score. Returns an empty list when the
272
- document has no parseable sections, so the caller chooses the fallback.
 
273
  """
274
  definitions = policy_definitions(policy_text)
275
  sections: list[tuple[str, list[str]]] = []
@@ -285,6 +314,8 @@ def parse_policy_markdown(policy_text: str, max_categories: int = POLICYLM_MAX_C
285
  sections.append(current)
286
 
287
  categories: list[dict] = []
 
 
288
  for name, body in sections:
289
  found: dict[str, str] = {}
290
  for raw in body:
@@ -302,9 +333,15 @@ def parse_policy_markdown(policy_text: str, max_categories: int = POLICYLM_MAX_C
302
  if context:
303
  entry["context"] = context
304
  categories.append(entry)
 
 
 
 
 
 
305
  if len(categories) >= max_categories:
306
  break
307
- return categories
308
 
309
 
310
  def policy_text_to_categories(policy_text: str, max_categories: int = POLICYLM_MAX_CATEGORIES) -> list[dict]:
@@ -315,6 +352,15 @@ def policy_text_to_categories(policy_text: str, max_categories: int = POLICYLM_M
315
  """
316
  categories = parse_policy_markdown(policy_text, max_categories)
317
  if categories:
 
 
 
 
 
 
 
 
 
318
  return categories
319
  flattened = _one_line(policy_text)
320
  return [{"name": "Policy violation", "violation_rule": flattened}] if flattened else []
 
118
 
119
  Each category maps directly onto policylm_infer.Category's constructor kwargs.
120
  Only non-empty optional fields are included so the handler can omit kwargs it
121
+ doesn't have values for. A rule with a "flag this" field is a scorable category; a
122
+ rule with only a "don't flag" / "exception" (an "applies to all" carve-out) is
123
+ policy-wide guidance and is folded into every real category rather than dropped,
124
+ so PolicyLM honours it like the prompted models do.
125
  """
126
  categories = []
127
+ global_dont: list[str] = []
128
+ global_exc: list[str] = []
129
+ for rule in nonempty_rules(rules):
130
  flag = _s(rule, "flag")
131
+ dont = _s(rule, "dont_flag")
132
+ exc = _s(rule, "exception")
133
  if not flag:
134
+ if dont:
135
+ global_dont.append(dont)
136
+ if exc:
137
+ global_exc.append(exc)
138
  continue
139
  context = _s(rule, "context")
140
  cat = {
141
+ "name": _s(rule, "name") or f"Rule {len(categories) + 1}",
142
  "violation_rule": f"{flag} ({context})" if context else flag,
143
  }
 
 
144
  if dont:
145
  cat["not_violation_rule"] = dont
146
  if exc:
147
  cat["exception_override"] = exc
148
  categories.append(cat)
149
+ return _fold_global(categories, _one_line(" ".join(global_dont)), _one_line(" ".join(global_exc)))
150
 
151
 
152
  def estimate_policy_tokens(rules: list[dict] | None) -> int:
 
270
  return " ".join(dict.fromkeys(kept))
271
 
272
 
273
+ def _fold_global(categories: list[dict], global_dont: str, global_exc: str) -> list[dict]:
274
+ """Merge policy-wide carve-outs into every category.
275
+
276
+ PolicyLM scores each category independently, so a "## Applies to all categories"
277
+ section — a Don't-flag / Exception clause with no Flag of its own — only takes
278
+ effect if it is present in each category. Without this it is dropped (nothing to
279
+ score), so PolicyLM flags content the carve-out protects while the prompted models,
280
+ reading the whole document, do not.
281
+ """
282
+ for cat in categories:
283
+ if global_dont:
284
+ cat["not_violation_rule"] = " ".join(
285
+ p for p in (cat.get("not_violation_rule", ""), global_dont) if p
286
+ )
287
+ if global_exc:
288
+ cat["exception_override"] = " ".join(
289
+ p for p in (cat.get("exception_override", ""), global_exc) if p
290
+ )
291
+ return categories
292
+
293
+
294
  def parse_policy_markdown(policy_text: str, max_categories: int = POLICYLM_MAX_CATEGORIES) -> list[dict]:
295
  """Parse a written policy into PolicyLM categories.
296
 
297
  Recognises the "## Name" + "Flag:/Don't flag:/Exception:" shape the workshop
298
+ policies use. A section with no Flag line is not its own category (nothing to
299
+ score) but, if it carries a Don't-flag / Exception clause (an "applies to all"
300
+ carve-out), that clause is folded into every real category rather than dropped.
301
+ Returns an empty list when the document has no parseable sections.
302
  """
303
  definitions = policy_definitions(policy_text)
304
  sections: list[tuple[str, list[str]]] = []
 
314
  sections.append(current)
315
 
316
  categories: list[dict] = []
317
+ global_dont: list[str] = []
318
+ global_exc: list[str] = []
319
  for name, body in sections:
320
  found: dict[str, str] = {}
321
  for raw in body:
 
333
  if context:
334
  entry["context"] = context
335
  categories.append(entry)
336
+ else:
337
+ # Flag-less section = policy-wide guidance; keep its carve-out clauses.
338
+ if found.get("not_violation_rule"):
339
+ global_dont.append(found["not_violation_rule"])
340
+ if found.get("exception_override"):
341
+ global_exc.append(found["exception_override"])
342
  if len(categories) >= max_categories:
343
  break
344
+ return _fold_global(categories, _one_line(" ".join(global_dont)), _one_line(" ".join(global_exc)))
345
 
346
 
347
  def policy_text_to_categories(policy_text: str, max_categories: int = POLICYLM_MAX_CATEGORIES) -> list[dict]:
 
352
  """
353
  categories = parse_policy_markdown(policy_text, max_categories)
354
  if categories:
355
+ # Fold the per-category definition context into the violation rule, the same way
356
+ # rules_to_categories does for the editor path. The endpoint handler only reads
357
+ # name/violation_rule/not_violation_rule/exception_override and ignores a separate
358
+ # "context" key, so on this free-form path (which bypasses the editor) the policy's
359
+ # preamble definitions would otherwise never reach PolicyLM.
360
+ for cat in categories:
361
+ context = cat.pop("context", "")
362
+ if context:
363
+ cat["violation_rule"] = f"{cat['violation_rule']} ({context})"
364
  return categories
365
  flattened = _one_line(policy_text)
366
  return [{"name": "Policy violation", "violation_rule": flattened}] if flattened else []