JSpon commited on
Commit
bca5542
·
verified ·
1 Parent(s): 72d7f4a

One route into the policy: add anything as rules

Browse files

The "structured rules vs policy text" switch asked people to understand an
internal distinction. Removed: the rule list is simply the policy. A preset
already arrives as rules, and "➕ Add as rules" turns anything written or pasted
into rules too — one per "## " section, or a single rule for a document with no
sections — which can then be edited or deleted. Nothing is silently overridden,
because there is only one way in.

Also carries a policy's preamble definitions into the rules. Lines of the form
"Protected classes: …" / "Not protected: …" sit before the first heading and were
dropped by the parser, so a model scored one category at a time never saw them
while a model handed the whole document did. They are now attached only to rules
whose text uses the term, which keeps the token cost down: hate speech picks up
definitions on 4 of 6 rules (~391 of 1662 tokens), mental health and scams on none.

This does not make PolicyLM honour scope exclusions. Measured on the hate-speech
policy, attacks on explicitly not-protected classes still flag: political ideology
0.945 -> 0.912, criminal history 0.974 -> 0.816, and routing the exclusion through
not_violation_rule instead of context does not help either (0.885 / 0.831). The
definitions move scores but never a verdict, which matches the earlier
inverted-policy result: permission-shaped instructions do not steer this model.

Files changed (3) hide show
  1. app.py +17 -24
  2. ui/tab_policy.py +49 -52
  3. utils/policy_rules.py +52 -3
app.py CHANGED
@@ -188,8 +188,6 @@ with gr.Blocks(title="Moderation Model Testing") as demo:
188
  structured_rules_state = policy_components["structured_rules_state"]
189
  rules_version_state = policy_components["rules_version_state"]
190
  policy_label_state = policy_components["policy_label_state"]
191
- policy_source_state = policy_components["policy_source_state"]
192
- effective_rules_state = policy_components["effective_rules_state"]
193
 
194
  config_components = build_config_tab()
195
  model_dropdown = config_components["model_dropdown"]
@@ -218,7 +216,7 @@ with gr.Blocks(title="Moderation Model Testing") as demo:
218
  inputs=[
219
  test_input,
220
  current_policy_state,
221
- effective_rules_state,
222
  model_dropdown,
223
  reasoning_effort,
224
  max_tokens,
@@ -262,7 +260,7 @@ with gr.Blocks(title="Moderation Model Testing") as demo:
262
  # --- Say which policy a run will use -------------------------------------------
263
  active_policy_display = testing_components["active_policy_display"]
264
 
265
- def _describe_active_policy(label, policy_text, rules, source):
266
  """One line naming the policy a run will actually use.
267
 
268
  Structured rules override the free-form text, so the count has to come from
@@ -270,7 +268,7 @@ with gr.Blocks(title="Moderation Model Testing") as demo:
270
  that is not what gets sent.
271
  """
272
  name = (label or "").strip() or "Custom (hand-written)"
273
- rules = (rules or []) if source == "Structured rules" else []
274
  on = [r for r in rules if rule_has_content(r) and rule_is_enabled(r)]
275
  off = [r for r in rules if rule_has_content(r) and not rule_is_enabled(r)]
276
  if on:
@@ -280,12 +278,10 @@ with gr.Blocks(title="Moderation Model Testing") as demo:
280
  return f"**Policy:** {name} — free-form text, {len((policy_text or '').strip()):,} characters"
281
  return "**Policy:** ⚠️ none loaded — set one on the 📋 Policy Definition tab"
282
 
283
- for _trigger in (current_policy_state, structured_rules_state,
284
- policy_label_state, policy_source_state):
285
  _trigger.change(
286
  _describe_active_policy,
287
- inputs=[policy_label_state, current_policy_state,
288
- structured_rules_state, policy_source_state],
289
  outputs=active_policy_display,
290
  )
291
 
@@ -297,20 +293,19 @@ with gr.Blocks(title="Moderation Model Testing") as demo:
297
  storage_key="roost_moderation_policy",
298
  )
299
 
300
- def _persist_policy(policy_text, rules, source):
301
- return {"policy": policy_text or "", "rules": rules or [], "source": source or "Policy text"}
302
 
303
  current_policy_state.change(
304
  _persist_policy,
305
- inputs=[current_policy_state, structured_rules_state, policy_source_state],
 
 
 
 
 
306
  outputs=policy_browser_state,
307
  )
308
- for _src in (structured_rules_state, policy_source_state):
309
- _src.change(
310
- _persist_policy,
311
- inputs=[current_policy_state, structured_rules_state, policy_source_state],
312
- outputs=policy_browser_state,
313
- )
314
 
315
  def _restore_policy(saved, version):
316
  """Put a saved policy back after a reload.
@@ -321,19 +316,17 @@ with gr.Blocks(title="Moderation Model Testing") as demo:
321
  the fresh defaults alone.
322
  """
323
  if not isinstance(saved, dict):
324
- return gr.skip(), gr.skip(), gr.skip(), gr.skip()
325
  policy_text = saved.get("policy") or ""
326
  rules = saved.get("rules") or []
327
  if not policy_text and not rules_have_content(rules):
328
- return gr.skip(), gr.skip(), gr.skip(), gr.skip()
329
- source = saved.get("source") or ("Structured rules" if rules else "Policy text")
330
- return policy_text, (rules or [new_rule()]), (version or 0) + 1, source
331
 
332
  demo.load(
333
  _restore_policy,
334
  inputs=[policy_browser_state, rules_version_state],
335
- outputs=[current_policy_state, structured_rules_state,
336
- rules_version_state, policy_source_state],
337
  )
338
 
339
  # Combined handler for login button click - updates all token-dependent UI
 
188
  structured_rules_state = policy_components["structured_rules_state"]
189
  rules_version_state = policy_components["rules_version_state"]
190
  policy_label_state = policy_components["policy_label_state"]
 
 
191
 
192
  config_components = build_config_tab()
193
  model_dropdown = config_components["model_dropdown"]
 
216
  inputs=[
217
  test_input,
218
  current_policy_state,
219
+ structured_rules_state,
220
  model_dropdown,
221
  reasoning_effort,
222
  max_tokens,
 
260
  # --- Say which policy a run will use -------------------------------------------
261
  active_policy_display = testing_components["active_policy_display"]
262
 
263
+ def _describe_active_policy(label, policy_text, rules):
264
  """One line naming the policy a run will actually use.
265
 
266
  Structured rules override the free-form text, so the count has to come from
 
268
  that is not what gets sent.
269
  """
270
  name = (label or "").strip() or "Custom (hand-written)"
271
+ rules = rules or []
272
  on = [r for r in rules if rule_has_content(r) and rule_is_enabled(r)]
273
  off = [r for r in rules if rule_has_content(r) and not rule_is_enabled(r)]
274
  if on:
 
278
  return f"**Policy:** {name} — free-form text, {len((policy_text or '').strip()):,} characters"
279
  return "**Policy:** ⚠️ none loaded — set one on the 📋 Policy Definition tab"
280
 
281
+ for _trigger in (current_policy_state, structured_rules_state, policy_label_state):
 
282
  _trigger.change(
283
  _describe_active_policy,
284
+ inputs=[policy_label_state, current_policy_state, structured_rules_state],
 
285
  outputs=active_policy_display,
286
  )
287
 
 
293
  storage_key="roost_moderation_policy",
294
  )
295
 
296
+ def _persist_policy(policy_text, rules):
297
+ return {"policy": policy_text or "", "rules": rules or []}
298
 
299
  current_policy_state.change(
300
  _persist_policy,
301
+ inputs=[current_policy_state, structured_rules_state],
302
+ outputs=policy_browser_state,
303
+ )
304
+ structured_rules_state.change(
305
+ _persist_policy,
306
+ inputs=[current_policy_state, structured_rules_state],
307
  outputs=policy_browser_state,
308
  )
 
 
 
 
 
 
309
 
310
  def _restore_policy(saved, version):
311
  """Put a saved policy back after a reload.
 
316
  the fresh defaults alone.
317
  """
318
  if not isinstance(saved, dict):
319
+ return gr.skip(), gr.skip(), gr.skip()
320
  policy_text = saved.get("policy") or ""
321
  rules = saved.get("rules") or []
322
  if not policy_text and not rules_have_content(rules):
323
+ return gr.skip(), gr.skip(), gr.skip()
324
+ return policy_text, (rules or [new_rule()]), (version or 0) + 1
 
325
 
326
  demo.load(
327
  _restore_policy,
328
  inputs=[policy_browser_state, rules_version_state],
329
+ outputs=[current_policy_state, structured_rules_state, rules_version_state],
 
330
  )
331
 
332
  # Combined handler for login button click - updates all token-dependent UI
ui/tab_policy.py CHANGED
@@ -10,6 +10,7 @@ sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
10
  from utils.helpers import load_policy_from_file, load_preset_policy, list_preset_policy_names
11
  from utils.policy_rules import (
12
  new_rule,
 
13
  parse_policy_markdown,
14
  rule_has_content,
15
  rule_is_enabled,
@@ -39,20 +40,7 @@ def build_policy_tab(base_dir: str, demo: gr.Blocks | None = None) -> dict:
39
  # hand-written). The Testing tab shows this so nobody runs content against a
40
  # policy they have forgotten they loaded.
41
  policy_label_state = gr.State(value="")
42
- # Which source is live. This used to be inferred — any rule with content beat
43
- # the free-form text — so a policy you typed or uploaded was silently ignored
44
- # whenever rules were loaded, with nothing on screen explaining why.
45
- policy_source_state = gr.State(value="Policy text")
46
- # Rules as the run should see them: empty when the text is the live source, so
47
- # the models are sent what the switch says rather than whatever is in the editor.
48
- effective_rules_state = gr.State(value=[])
49
 
50
- policy_source_radio = gr.Radio(
51
- label="Which policy is live?",
52
- choices=["Structured rules", "Policy text"],
53
- value="Policy text",
54
- info="Loading a preset switches to rules; typing or uploading switches to text.",
55
- )
56
 
57
  # --- Structured rule editor (model-agnostic; native where supported) -------------------------------
58
  with gr.Accordion("🧩 Structured Rules (recommended — one score per rule)", open=True):
@@ -193,16 +181,12 @@ def build_policy_tab(base_dir: str, demo: gr.Blocks | None = None) -> dict:
193
  outputs=[structured_rules_state, rules_version_state],
194
  )
195
 
196
- def sync_rules_to_policy(rules, current_policy, source):
197
- """Render the rules into the policy text, but only when they are live."""
198
- if source == "Structured rules" and rules_have_content(rules):
199
  return render_rules_to_text(rules)
200
  return current_policy
201
 
202
- def compute_effective_rules(rules, source):
203
- """What the models should actually receive for the structured path."""
204
- return rules if source == "Structured rules" else []
205
-
206
  # Live limit warning + keep the effective policy text in sync.
207
  structured_rules_state.change(
208
  policy_limits_markdown,
@@ -211,21 +195,9 @@ def build_policy_tab(base_dir: str, demo: gr.Blocks | None = None) -> dict:
211
  )
212
  structured_rules_state.change(
213
  sync_rules_to_policy,
214
- inputs=[structured_rules_state, current_policy_state, policy_source_state],
215
  outputs=current_policy_state,
216
  )
217
- for _src in (structured_rules_state, policy_source_state):
218
- _src.change(
219
- compute_effective_rules,
220
- inputs=[structured_rules_state, policy_source_state],
221
- outputs=effective_rules_state,
222
- )
223
- policy_source_radio.change(
224
- lambda v: v, inputs=policy_source_radio, outputs=policy_source_state
225
- )
226
- policy_source_state.change(
227
- lambda v: gr.update(value=v), inputs=policy_source_state, outputs=policy_source_radio
228
- )
229
 
230
  # Existing Policy Accordion
231
  with gr.Accordion("📥 Load Existing Policy", open=False):
@@ -247,9 +219,16 @@ def build_policy_tab(base_dir: str, demo: gr.Blocks | None = None) -> dict:
247
  with gr.Accordion("✏️ Manual Editing", open=False):
248
  manual_text = gr.Textbox(
249
  label="Policy Text",
250
- placeholder="Enter or edit policy markdown...",
251
  lines=20
252
  )
 
 
 
 
 
 
 
253
 
254
  policy_preview = gr.Markdown(value="*No policy loaded*")
255
 
@@ -265,7 +244,7 @@ def build_policy_tab(base_dir: str, demo: gr.Blocks | None = None) -> dict:
265
  both in the same list, each tagged with where it came from, so a rule you
266
  added sits beside the preset's and can be compared, switched off or edited.
267
  """
268
- blank = ("", "*No policy loaded*", "", [new_rule()], (version or 0) + 1, "", "Policy text")
269
  if not name:
270
  return blank
271
 
@@ -285,14 +264,13 @@ def build_policy_tab(base_dir: str, demo: gr.Blocks | None = None) -> dict:
285
  "enabled": True, "source": name}
286
  for c in parsed
287
  ] or [new_rule()]
288
- return policy_text, policy_text, policy_text, rules, (version or 0) + 1, name, "Structured rules"
289
 
290
  load_preset_btn.click(
291
  load_preset_handler,
292
  inputs=[preset_dropdown, uploaded_policies_state, rules_version_state],
293
  outputs=[current_policy_state, manual_text, policy_preview,
294
- structured_rules_state, rules_version_state, policy_label_state,
295
- policy_source_state],
296
  )
297
 
298
  def load_upload_handler(f, uploaded_policies):
@@ -337,33 +315,54 @@ def build_policy_tab(base_dir: str, demo: gr.Blocks | None = None) -> dict:
337
  load_upload_handler,
338
  inputs=[upload_file, uploaded_policies_state],
339
  outputs=[current_policy_state, manual_text, policy_preview, preset_dropdown, uploaded_policies_state],
340
- ).then(
341
- # An uploaded document is text, not rules, so make the text live —
342
- # otherwise rules left over from a preset would keep overriding it.
343
- lambda: ("Policy text", "Uploaded policy"),
344
- outputs=[policy_source_state, policy_label_state],
345
  )
346
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
347
  def update_preview(text):
348
  return text if text else "*No policy loaded*"
349
 
350
  # Update state and preview when user leaves the textbox (blur event)
 
 
 
 
 
 
351
  manual_text.blur(
352
- # Editing the text makes the text live, otherwise loaded rules would keep
353
- # overwriting what was just typed.
354
- lambda t: (t, update_preview(t), "Policy text", "Custom (hand-written)"),
 
355
  inputs=manual_text,
356
- outputs=[current_policy_state, policy_preview, policy_source_state, policy_label_state],
357
  )
358
 
359
  clear_policy_btn.click(
360
  # Clear the rules too: clearing only the text left loaded rules live, so
361
  # the policy was not actually cleared.
362
- lambda v: ("", "", "*No policy loaded*", [new_rule()], (v or 0) + 1, "", "Policy text"),
363
  inputs=rules_version_state,
364
  outputs=[current_policy_state, manual_text, policy_preview,
365
- structured_rules_state, rules_version_state, policy_label_state,
366
- policy_source_state],
367
  )
368
 
369
  # Sync UI components when state changes externally (e.g., from dataset load)
@@ -383,6 +382,4 @@ def build_policy_tab(base_dir: str, demo: gr.Blocks | None = None) -> dict:
383
  # Exposed so restoring a saved policy can force the rule editor to re-render.
384
  "rules_version_state": rules_version_state,
385
  "policy_label_state": policy_label_state,
386
- "policy_source_state": policy_source_state,
387
- "effective_rules_state": effective_rules_state,
388
  }
 
10
  from utils.helpers import load_policy_from_file, load_preset_policy, list_preset_policy_names
11
  from utils.policy_rules import (
12
  new_rule,
13
+ policy_text_to_categories,
14
  parse_policy_markdown,
15
  rule_has_content,
16
  rule_is_enabled,
 
40
  # hand-written). The Testing tab shows this so nobody runs content against a
41
  # policy they have forgotten they loaded.
42
  policy_label_state = gr.State(value="")
 
 
 
 
 
 
 
43
 
 
 
 
 
 
 
44
 
45
  # --- Structured rule editor (model-agnostic; native where supported) -------------------------------
46
  with gr.Accordion("🧩 Structured Rules (recommended — one score per rule)", open=True):
 
181
  outputs=[structured_rules_state, rules_version_state],
182
  )
183
 
184
+ def sync_rules_to_policy(rules, current_policy):
185
+ """The rule list is the policy; render it for models that read text."""
186
+ if rules_have_content(rules):
187
  return render_rules_to_text(rules)
188
  return current_policy
189
 
 
 
 
 
190
  # Live limit warning + keep the effective policy text in sync.
191
  structured_rules_state.change(
192
  policy_limits_markdown,
 
195
  )
196
  structured_rules_state.change(
197
  sync_rules_to_policy,
198
+ inputs=[structured_rules_state, current_policy_state],
199
  outputs=current_policy_state,
200
  )
 
 
 
 
 
 
 
 
 
 
 
 
201
 
202
  # Existing Policy Accordion
203
  with gr.Accordion("📥 Load Existing Policy", open=False):
 
219
  with gr.Accordion("✏️ Manual Editing", open=False):
220
  manual_text = gr.Textbox(
221
  label="Policy Text",
222
+ placeholder="Write or paste a policy, then add it as rules...",
223
  lines=20
224
  )
225
+ add_as_rules_btn = gr.Button("➕ Add as rules", variant="primary")
226
+ gr.Markdown(
227
+ "Turns whatever is in the box into rules and appends them to the list "
228
+ "above. A policy written as `## Name` with `Flag:` / `Don't flag:` / "
229
+ "`Exception:` lines becomes one rule per section; anything else becomes "
230
+ "a single rule you can then edit or delete."
231
+ )
232
 
233
  policy_preview = gr.Markdown(value="*No policy loaded*")
234
 
 
244
  both in the same list, each tagged with where it came from, so a rule you
245
  added sits beside the preset's and can be compared, switched off or edited.
246
  """
247
+ blank = ("", "*No policy loaded*", "", [new_rule()], (version or 0) + 1, "")
248
  if not name:
249
  return blank
250
 
 
264
  "enabled": True, "source": name}
265
  for c in parsed
266
  ] or [new_rule()]
267
+ return policy_text, policy_text, policy_text, rules, (version or 0) + 1, name
268
 
269
  load_preset_btn.click(
270
  load_preset_handler,
271
  inputs=[preset_dropdown, uploaded_policies_state, rules_version_state],
272
  outputs=[current_policy_state, manual_text, policy_preview,
273
+ structured_rules_state, rules_version_state, policy_label_state],
 
274
  )
275
 
276
  def load_upload_handler(f, uploaded_policies):
 
315
  load_upload_handler,
316
  inputs=[upload_file, uploaded_policies_state],
317
  outputs=[current_policy_state, manual_text, policy_preview, preset_dropdown, uploaded_policies_state],
 
 
 
 
 
318
  )
319
 
320
+ def add_as_rules_handler(text, rules, version, label):
321
+ """Append whatever is in the text box to the rule list.
322
+
323
+ One button for both routes: a preset already arrives as rules, and this
324
+ turns anything written or pasted into rules too, so the list is always the
325
+ policy and nothing is silently overridden by something typed elsewhere.
326
+ """
327
+ if not (text or "").strip():
328
+ return rules, version, label, gr.update()
329
+ parsed = policy_text_to_categories(text)
330
+ added = [
331
+ {"name": c.get("name", ""), "flag": c.get("violation_rule", ""),
332
+ "dont_flag": c.get("not_violation_rule", ""),
333
+ "exception": c.get("exception_override", ""),
334
+ "context": c.get("context", ""), "enabled": True, "source": ""}
335
+ for c in parsed
336
+ ]
337
+ kept = [r for r in (rules or []) if rule_has_content(r)]
338
+ return kept + added, (version or 0) + 1, (label or "Custom (hand-written)"), gr.update(value="")
339
+
340
  def update_preview(text):
341
  return text if text else "*No policy loaded*"
342
 
343
  # Update state and preview when user leaves the textbox (blur event)
344
+ add_as_rules_btn.click(
345
+ add_as_rules_handler,
346
+ inputs=[manual_text, structured_rules_state, rules_version_state, policy_label_state],
347
+ outputs=[structured_rules_state, rules_version_state, policy_label_state, manual_text],
348
+ )
349
+
350
  manual_text.blur(
351
+ # Keeps the preview in step while writing. The text only becomes the policy
352
+ # when "Add as rules" is pressed, so there is one route in and no silent
353
+ # override of the rule list.
354
+ lambda t: update_preview(t),
355
  inputs=manual_text,
356
+ outputs=policy_preview,
357
  )
358
 
359
  clear_policy_btn.click(
360
  # Clear the rules too: clearing only the text left loaded rules live, so
361
  # the policy was not actually cleared.
362
+ lambda v: ("", "", "*No policy loaded*", [new_rule()], (v or 0) + 1, ""),
363
  inputs=rules_version_state,
364
  outputs=[current_policy_state, manual_text, policy_preview,
365
+ structured_rules_state, rules_version_state, policy_label_state],
 
366
  )
367
 
368
  # Sync UI components when state changes externally (e.g., from dataset load)
 
382
  # Exposed so restoring a saved policy can force the rule editor to re-render.
383
  "rules_version_state": rules_version_state,
384
  "policy_label_state": policy_label_state,
 
 
385
  }
utils/policy_rules.py CHANGED
@@ -25,7 +25,7 @@ import math
25
  from utils.constants import POLICYLM_MAX_CATEGORIES, POLICYLM_MAX_POLICY_TOKENS
26
 
27
  # The empty-rule template. Copy it (`dict(EMPTY_RULE)`) — never mutate the shared one.
28
- EMPTY_RULE = {"name": "", "flag": "", "dont_flag": "", "exception": "", "enabled": True}
29
 
30
  _FIELD_ORDER = ("name", "flag", "dont_flag", "exception")
31
 
@@ -91,6 +91,11 @@ def render_rules_to_text(rules: list[dict] | None) -> str:
91
  "**Exception**. Evaluate each rule independently.",
92
  "",
93
  ]
 
 
 
 
 
94
  for i, rule in enumerate(kept, start=1):
95
  name = _s(rule, "name") or f"Rule {i}"
96
  lines.append(f"## {i}. {name}")
@@ -122,9 +127,10 @@ def rules_to_categories(rules: list[dict] | None) -> list[dict]:
122
  if not flag:
123
  # Without a violation rule there is nothing for PolicyLM to score.
124
  continue
 
125
  cat = {
126
  "name": _s(rule, "name") or f"Rule {i}",
127
- "violation_rule": flag,
128
  }
129
  dont = _s(rule, "dont_flag")
130
  exc = _s(rule, "exception")
@@ -219,6 +225,44 @@ def _one_line(text: str) -> str:
219
  return " ".join(str(text).split())
220
 
221
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
222
  def parse_policy_markdown(policy_text: str, max_categories: int = POLICYLM_MAX_CATEGORIES) -> list[dict]:
223
  """Parse a written policy into PolicyLM categories.
224
 
@@ -227,6 +271,7 @@ def parse_policy_markdown(policy_text: str, max_categories: int = POLICYLM_MAX_C
227
  violation rule is nothing for PolicyLM to score. Returns an empty list when the
228
  document has no parseable sections, so the caller chooses the fallback.
229
  """
 
230
  sections: list[tuple[str, list[str]]] = []
231
  current: tuple[str, list[str]] | None = None
232
  for line in (policy_text or "").splitlines():
@@ -252,7 +297,11 @@ def parse_policy_markdown(policy_text: str, max_categories: int = POLICYLM_MAX_C
252
  if value:
253
  found[field] = value
254
  if "violation_rule" in found:
255
- categories.append({"name": _one_line(name)[:120], **found})
 
 
 
 
256
  if len(categories) >= max_categories:
257
  break
258
  return categories
 
25
  from utils.constants import POLICYLM_MAX_CATEGORIES, POLICYLM_MAX_POLICY_TOKENS
26
 
27
  # The empty-rule template. Copy it (`dict(EMPTY_RULE)`) — never mutate the shared one.
28
+ EMPTY_RULE = {"name": "", "flag": "", "dont_flag": "", "exception": "", "enabled": True, "context": ""}
29
 
30
  _FIELD_ORDER = ("name", "flag", "dont_flag", "exception")
31
 
 
91
  "**Exception**. Evaluate each rule independently.",
92
  "",
93
  ]
94
+ seen_context = list(dict.fromkeys(_s(r, "context") for r in kept if _s(r, "context")))
95
+ if seen_context:
96
+ # Once at the top for models that read the whole document; the per-rule copies
97
+ # exist only for models scored a category at a time.
98
+ lines += ["**Definitions and scope:** " + " ".join(seen_context), ""]
99
  for i, rule in enumerate(kept, start=1):
100
  name = _s(rule, "name") or f"Rule {i}"
101
  lines.append(f"## {i}. {name}")
 
127
  if not flag:
128
  # Without a violation rule there is nothing for PolicyLM to score.
129
  continue
130
+ context = _s(rule, "context")
131
  cat = {
132
  "name": _s(rule, "name") or f"Rule {i}",
133
+ "violation_rule": f"{flag} ({context})" if context else flag,
134
  }
135
  dont = _s(rule, "dont_flag")
136
  exc = _s(rule, "exception")
 
225
  return " ".join(str(text).split())
226
 
227
 
228
+ def policy_definitions(policy_text: str) -> list[tuple[str, str]]:
229
+ """Definitions stated before the first "## " heading, as (term, line) pairs.
230
+
231
+ A policy states its scope once at the top — "Protected classes: …",
232
+ "Not protected: …". A model handed the whole document reads that; a model scored
233
+ one category at a time never sees it, which measurably changes verdicts.
234
+
235
+ Only lines of the form "Term: definition" count. Prose in the preamble is left for
236
+ the rendered document, since there is no term to attach it to.
237
+ """
238
+ head = (policy_text or "").split("\n## ")[0]
239
+ out: list[tuple[str, str]] = []
240
+ for raw in head.splitlines():
241
+ line = raw.strip().lstrip("-* ").strip()
242
+ if line.startswith("#") or ":" not in line:
243
+ continue
244
+ term, _, rest = line.partition(":")
245
+ if term and rest.strip() and len(term.split()) <= 4:
246
+ out.append((_one_line(term).lower(), _one_line(line)))
247
+ return out
248
+
249
+
250
+ def _relevant_definitions(text: str, definitions: list[tuple[str, str]]) -> str:
251
+ """The definitions this rule actually relies on.
252
+
253
+ Attached per rule rather than to all of them: repeating every definition on every
254
+ category burns the token budget and tells a model about terms its rule never uses.
255
+ A term matches on its longest word ("protected" for "Protected classes"), which
256
+ also catches the singular.
257
+ """
258
+ lowered = (text or "").lower()
259
+ kept = [
260
+ line for term, line in definitions
261
+ if any(w for w in sorted(term.split(), key=len, reverse=True)[:1] if w in lowered)
262
+ ]
263
+ return " ".join(dict.fromkeys(kept))
264
+
265
+
266
  def parse_policy_markdown(policy_text: str, max_categories: int = POLICYLM_MAX_CATEGORIES) -> list[dict]:
267
  """Parse a written policy into PolicyLM categories.
268
 
 
271
  violation rule is nothing for PolicyLM to score. Returns an empty list when the
272
  document has no parseable sections, so the caller chooses the fallback.
273
  """
274
+ definitions = policy_definitions(policy_text)
275
  sections: list[tuple[str, list[str]]] = []
276
  current: tuple[str, list[str]] | None = None
277
  for line in (policy_text or "").splitlines():
 
297
  if value:
298
  found[field] = value
299
  if "violation_rule" in found:
300
+ entry = {"name": _one_line(name)[:120], **found}
301
+ context = _relevant_definitions(" ".join([name, *found.values()]), definitions)
302
+ if context:
303
+ entry["context"] = context
304
+ categories.append(entry)
305
  if len(categories) >= max_categories:
306
  break
307
  return categories