One route into the policy: add anything as rules
Browse filesThe "structured rules vs policy text" switch asked people to understand an
internal distinction. Removed: the rule list is simply the policy. A preset
already arrives as rules, and "➕ Add as rules" turns anything written or pasted
into rules too — one per "## " section, or a single rule for a document with no
sections — which can then be edited or deleted. Nothing is silently overridden,
because there is only one way in.
Also carries a policy's preamble definitions into the rules. Lines of the form
"Protected classes: …" / "Not protected: …" sit before the first heading and were
dropped by the parser, so a model scored one category at a time never saw them
while a model handed the whole document did. They are now attached only to rules
whose text uses the term, which keeps the token cost down: hate speech picks up
definitions on 4 of 6 rules (~391 of 1662 tokens), mental health and scams on none.
This does not make PolicyLM honour scope exclusions. Measured on the hate-speech
policy, attacks on explicitly not-protected classes still flag: political ideology
0.945 -> 0.912, criminal history 0.974 -> 0.816, and routing the exclusion through
not_violation_rule instead of context does not help either (0.885 / 0.831). The
definitions move scores but never a verdict, which matches the earlier
inverted-policy result: permission-shaped instructions do not steer this model.
- app.py +17 -24
- ui/tab_policy.py +49 -52
- utils/policy_rules.py +52 -3
|
@@ -188,8 +188,6 @@ with gr.Blocks(title="Moderation Model Testing") as demo:
|
|
| 188 |
structured_rules_state = policy_components["structured_rules_state"]
|
| 189 |
rules_version_state = policy_components["rules_version_state"]
|
| 190 |
policy_label_state = policy_components["policy_label_state"]
|
| 191 |
-
policy_source_state = policy_components["policy_source_state"]
|
| 192 |
-
effective_rules_state = policy_components["effective_rules_state"]
|
| 193 |
|
| 194 |
config_components = build_config_tab()
|
| 195 |
model_dropdown = config_components["model_dropdown"]
|
|
@@ -218,7 +216,7 @@ with gr.Blocks(title="Moderation Model Testing") as demo:
|
|
| 218 |
inputs=[
|
| 219 |
test_input,
|
| 220 |
current_policy_state,
|
| 221 |
-
|
| 222 |
model_dropdown,
|
| 223 |
reasoning_effort,
|
| 224 |
max_tokens,
|
|
@@ -262,7 +260,7 @@ with gr.Blocks(title="Moderation Model Testing") as demo:
|
|
| 262 |
# --- Say which policy a run will use -------------------------------------------
|
| 263 |
active_policy_display = testing_components["active_policy_display"]
|
| 264 |
|
| 265 |
-
def _describe_active_policy(label, policy_text, rules
|
| 266 |
"""One line naming the policy a run will actually use.
|
| 267 |
|
| 268 |
Structured rules override the free-form text, so the count has to come from
|
|
@@ -270,7 +268,7 @@ with gr.Blocks(title="Moderation Model Testing") as demo:
|
|
| 270 |
that is not what gets sent.
|
| 271 |
"""
|
| 272 |
name = (label or "").strip() or "Custom (hand-written)"
|
| 273 |
-
rules =
|
| 274 |
on = [r for r in rules if rule_has_content(r) and rule_is_enabled(r)]
|
| 275 |
off = [r for r in rules if rule_has_content(r) and not rule_is_enabled(r)]
|
| 276 |
if on:
|
|
@@ -280,12 +278,10 @@ with gr.Blocks(title="Moderation Model Testing") as demo:
|
|
| 280 |
return f"**Policy:** {name} — free-form text, {len((policy_text or '').strip()):,} characters"
|
| 281 |
return "**Policy:** ⚠️ none loaded — set one on the 📋 Policy Definition tab"
|
| 282 |
|
| 283 |
-
for _trigger in (current_policy_state, structured_rules_state,
|
| 284 |
-
policy_label_state, policy_source_state):
|
| 285 |
_trigger.change(
|
| 286 |
_describe_active_policy,
|
| 287 |
-
inputs=[policy_label_state, current_policy_state,
|
| 288 |
-
structured_rules_state, policy_source_state],
|
| 289 |
outputs=active_policy_display,
|
| 290 |
)
|
| 291 |
|
|
@@ -297,20 +293,19 @@ with gr.Blocks(title="Moderation Model Testing") as demo:
|
|
| 297 |
storage_key="roost_moderation_policy",
|
| 298 |
)
|
| 299 |
|
| 300 |
-
def _persist_policy(policy_text, rules
|
| 301 |
-
return {"policy": policy_text or "", "rules": rules or []
|
| 302 |
|
| 303 |
current_policy_state.change(
|
| 304 |
_persist_policy,
|
| 305 |
-
inputs=[current_policy_state, structured_rules_state
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 306 |
outputs=policy_browser_state,
|
| 307 |
)
|
| 308 |
-
for _src in (structured_rules_state, policy_source_state):
|
| 309 |
-
_src.change(
|
| 310 |
-
_persist_policy,
|
| 311 |
-
inputs=[current_policy_state, structured_rules_state, policy_source_state],
|
| 312 |
-
outputs=policy_browser_state,
|
| 313 |
-
)
|
| 314 |
|
| 315 |
def _restore_policy(saved, version):
|
| 316 |
"""Put a saved policy back after a reload.
|
|
@@ -321,19 +316,17 @@ with gr.Blocks(title="Moderation Model Testing") as demo:
|
|
| 321 |
the fresh defaults alone.
|
| 322 |
"""
|
| 323 |
if not isinstance(saved, dict):
|
| 324 |
-
return gr.skip(), gr.skip(), gr.skip()
|
| 325 |
policy_text = saved.get("policy") or ""
|
| 326 |
rules = saved.get("rules") or []
|
| 327 |
if not policy_text and not rules_have_content(rules):
|
| 328 |
-
return gr.skip(), gr.skip(), gr.skip()
|
| 329 |
-
|
| 330 |
-
return policy_text, (rules or [new_rule()]), (version or 0) + 1, source
|
| 331 |
|
| 332 |
demo.load(
|
| 333 |
_restore_policy,
|
| 334 |
inputs=[policy_browser_state, rules_version_state],
|
| 335 |
-
outputs=[current_policy_state, structured_rules_state,
|
| 336 |
-
rules_version_state, policy_source_state],
|
| 337 |
)
|
| 338 |
|
| 339 |
# Combined handler for login button click - updates all token-dependent UI
|
|
|
|
| 188 |
structured_rules_state = policy_components["structured_rules_state"]
|
| 189 |
rules_version_state = policy_components["rules_version_state"]
|
| 190 |
policy_label_state = policy_components["policy_label_state"]
|
|
|
|
|
|
|
| 191 |
|
| 192 |
config_components = build_config_tab()
|
| 193 |
model_dropdown = config_components["model_dropdown"]
|
|
|
|
| 216 |
inputs=[
|
| 217 |
test_input,
|
| 218 |
current_policy_state,
|
| 219 |
+
structured_rules_state,
|
| 220 |
model_dropdown,
|
| 221 |
reasoning_effort,
|
| 222 |
max_tokens,
|
|
|
|
| 260 |
# --- Say which policy a run will use -------------------------------------------
|
| 261 |
active_policy_display = testing_components["active_policy_display"]
|
| 262 |
|
| 263 |
+
def _describe_active_policy(label, policy_text, rules):
|
| 264 |
"""One line naming the policy a run will actually use.
|
| 265 |
|
| 266 |
Structured rules override the free-form text, so the count has to come from
|
|
|
|
| 268 |
that is not what gets sent.
|
| 269 |
"""
|
| 270 |
name = (label or "").strip() or "Custom (hand-written)"
|
| 271 |
+
rules = rules or []
|
| 272 |
on = [r for r in rules if rule_has_content(r) and rule_is_enabled(r)]
|
| 273 |
off = [r for r in rules if rule_has_content(r) and not rule_is_enabled(r)]
|
| 274 |
if on:
|
|
|
|
| 278 |
return f"**Policy:** {name} — free-form text, {len((policy_text or '').strip()):,} characters"
|
| 279 |
return "**Policy:** ⚠️ none loaded — set one on the 📋 Policy Definition tab"
|
| 280 |
|
| 281 |
+
for _trigger in (current_policy_state, structured_rules_state, policy_label_state):
|
|
|
|
| 282 |
_trigger.change(
|
| 283 |
_describe_active_policy,
|
| 284 |
+
inputs=[policy_label_state, current_policy_state, structured_rules_state],
|
|
|
|
| 285 |
outputs=active_policy_display,
|
| 286 |
)
|
| 287 |
|
|
|
|
| 293 |
storage_key="roost_moderation_policy",
|
| 294 |
)
|
| 295 |
|
| 296 |
+
def _persist_policy(policy_text, rules):
|
| 297 |
+
return {"policy": policy_text or "", "rules": rules or []}
|
| 298 |
|
| 299 |
current_policy_state.change(
|
| 300 |
_persist_policy,
|
| 301 |
+
inputs=[current_policy_state, structured_rules_state],
|
| 302 |
+
outputs=policy_browser_state,
|
| 303 |
+
)
|
| 304 |
+
structured_rules_state.change(
|
| 305 |
+
_persist_policy,
|
| 306 |
+
inputs=[current_policy_state, structured_rules_state],
|
| 307 |
outputs=policy_browser_state,
|
| 308 |
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 309 |
|
| 310 |
def _restore_policy(saved, version):
|
| 311 |
"""Put a saved policy back after a reload.
|
|
|
|
| 316 |
the fresh defaults alone.
|
| 317 |
"""
|
| 318 |
if not isinstance(saved, dict):
|
| 319 |
+
return gr.skip(), gr.skip(), gr.skip()
|
| 320 |
policy_text = saved.get("policy") or ""
|
| 321 |
rules = saved.get("rules") or []
|
| 322 |
if not policy_text and not rules_have_content(rules):
|
| 323 |
+
return gr.skip(), gr.skip(), gr.skip()
|
| 324 |
+
return policy_text, (rules or [new_rule()]), (version or 0) + 1
|
|
|
|
| 325 |
|
| 326 |
demo.load(
|
| 327 |
_restore_policy,
|
| 328 |
inputs=[policy_browser_state, rules_version_state],
|
| 329 |
+
outputs=[current_policy_state, structured_rules_state, rules_version_state],
|
|
|
|
| 330 |
)
|
| 331 |
|
| 332 |
# Combined handler for login button click - updates all token-dependent UI
|
|
@@ -10,6 +10,7 @@ sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
|
| 10 |
from utils.helpers import load_policy_from_file, load_preset_policy, list_preset_policy_names
|
| 11 |
from utils.policy_rules import (
|
| 12 |
new_rule,
|
|
|
|
| 13 |
parse_policy_markdown,
|
| 14 |
rule_has_content,
|
| 15 |
rule_is_enabled,
|
|
@@ -39,20 +40,7 @@ def build_policy_tab(base_dir: str, demo: gr.Blocks | None = None) -> dict:
|
|
| 39 |
# hand-written). The Testing tab shows this so nobody runs content against a
|
| 40 |
# policy they have forgotten they loaded.
|
| 41 |
policy_label_state = gr.State(value="")
|
| 42 |
-
# Which source is live. This used to be inferred — any rule with content beat
|
| 43 |
-
# the free-form text — so a policy you typed or uploaded was silently ignored
|
| 44 |
-
# whenever rules were loaded, with nothing on screen explaining why.
|
| 45 |
-
policy_source_state = gr.State(value="Policy text")
|
| 46 |
-
# Rules as the run should see them: empty when the text is the live source, so
|
| 47 |
-
# the models are sent what the switch says rather than whatever is in the editor.
|
| 48 |
-
effective_rules_state = gr.State(value=[])
|
| 49 |
|
| 50 |
-
policy_source_radio = gr.Radio(
|
| 51 |
-
label="Which policy is live?",
|
| 52 |
-
choices=["Structured rules", "Policy text"],
|
| 53 |
-
value="Policy text",
|
| 54 |
-
info="Loading a preset switches to rules; typing or uploading switches to text.",
|
| 55 |
-
)
|
| 56 |
|
| 57 |
# --- Structured rule editor (model-agnostic; native where supported) -------------------------------
|
| 58 |
with gr.Accordion("🧩 Structured Rules (recommended — one score per rule)", open=True):
|
|
@@ -193,16 +181,12 @@ def build_policy_tab(base_dir: str, demo: gr.Blocks | None = None) -> dict:
|
|
| 193 |
outputs=[structured_rules_state, rules_version_state],
|
| 194 |
)
|
| 195 |
|
| 196 |
-
def sync_rules_to_policy(rules, current_policy
|
| 197 |
-
"""
|
| 198 |
-
if
|
| 199 |
return render_rules_to_text(rules)
|
| 200 |
return current_policy
|
| 201 |
|
| 202 |
-
def compute_effective_rules(rules, source):
|
| 203 |
-
"""What the models should actually receive for the structured path."""
|
| 204 |
-
return rules if source == "Structured rules" else []
|
| 205 |
-
|
| 206 |
# Live limit warning + keep the effective policy text in sync.
|
| 207 |
structured_rules_state.change(
|
| 208 |
policy_limits_markdown,
|
|
@@ -211,21 +195,9 @@ def build_policy_tab(base_dir: str, demo: gr.Blocks | None = None) -> dict:
|
|
| 211 |
)
|
| 212 |
structured_rules_state.change(
|
| 213 |
sync_rules_to_policy,
|
| 214 |
-
inputs=[structured_rules_state, current_policy_state
|
| 215 |
outputs=current_policy_state,
|
| 216 |
)
|
| 217 |
-
for _src in (structured_rules_state, policy_source_state):
|
| 218 |
-
_src.change(
|
| 219 |
-
compute_effective_rules,
|
| 220 |
-
inputs=[structured_rules_state, policy_source_state],
|
| 221 |
-
outputs=effective_rules_state,
|
| 222 |
-
)
|
| 223 |
-
policy_source_radio.change(
|
| 224 |
-
lambda v: v, inputs=policy_source_radio, outputs=policy_source_state
|
| 225 |
-
)
|
| 226 |
-
policy_source_state.change(
|
| 227 |
-
lambda v: gr.update(value=v), inputs=policy_source_state, outputs=policy_source_radio
|
| 228 |
-
)
|
| 229 |
|
| 230 |
# Existing Policy Accordion
|
| 231 |
with gr.Accordion("📥 Load Existing Policy", open=False):
|
|
@@ -247,9 +219,16 @@ def build_policy_tab(base_dir: str, demo: gr.Blocks | None = None) -> dict:
|
|
| 247 |
with gr.Accordion("✏️ Manual Editing", open=False):
|
| 248 |
manual_text = gr.Textbox(
|
| 249 |
label="Policy Text",
|
| 250 |
-
placeholder="
|
| 251 |
lines=20
|
| 252 |
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 253 |
|
| 254 |
policy_preview = gr.Markdown(value="*No policy loaded*")
|
| 255 |
|
|
@@ -265,7 +244,7 @@ def build_policy_tab(base_dir: str, demo: gr.Blocks | None = None) -> dict:
|
|
| 265 |
both in the same list, each tagged with where it came from, so a rule you
|
| 266 |
added sits beside the preset's and can be compared, switched off or edited.
|
| 267 |
"""
|
| 268 |
-
blank = ("", "*No policy loaded*", "", [new_rule()], (version or 0) + 1, ""
|
| 269 |
if not name:
|
| 270 |
return blank
|
| 271 |
|
|
@@ -285,14 +264,13 @@ def build_policy_tab(base_dir: str, demo: gr.Blocks | None = None) -> dict:
|
|
| 285 |
"enabled": True, "source": name}
|
| 286 |
for c in parsed
|
| 287 |
] or [new_rule()]
|
| 288 |
-
return policy_text, policy_text, policy_text, rules, (version or 0) + 1, name
|
| 289 |
|
| 290 |
load_preset_btn.click(
|
| 291 |
load_preset_handler,
|
| 292 |
inputs=[preset_dropdown, uploaded_policies_state, rules_version_state],
|
| 293 |
outputs=[current_policy_state, manual_text, policy_preview,
|
| 294 |
-
structured_rules_state, rules_version_state, policy_label_state,
|
| 295 |
-
policy_source_state],
|
| 296 |
)
|
| 297 |
|
| 298 |
def load_upload_handler(f, uploaded_policies):
|
|
@@ -337,33 +315,54 @@ def build_policy_tab(base_dir: str, demo: gr.Blocks | None = None) -> dict:
|
|
| 337 |
load_upload_handler,
|
| 338 |
inputs=[upload_file, uploaded_policies_state],
|
| 339 |
outputs=[current_policy_state, manual_text, policy_preview, preset_dropdown, uploaded_policies_state],
|
| 340 |
-
).then(
|
| 341 |
-
# An uploaded document is text, not rules, so make the text live —
|
| 342 |
-
# otherwise rules left over from a preset would keep overriding it.
|
| 343 |
-
lambda: ("Policy text", "Uploaded policy"),
|
| 344 |
-
outputs=[policy_source_state, policy_label_state],
|
| 345 |
)
|
| 346 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 347 |
def update_preview(text):
|
| 348 |
return text if text else "*No policy loaded*"
|
| 349 |
|
| 350 |
# Update state and preview when user leaves the textbox (blur event)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 351 |
manual_text.blur(
|
| 352 |
-
#
|
| 353 |
-
#
|
| 354 |
-
|
|
|
|
| 355 |
inputs=manual_text,
|
| 356 |
-
outputs=
|
| 357 |
)
|
| 358 |
|
| 359 |
clear_policy_btn.click(
|
| 360 |
# Clear the rules too: clearing only the text left loaded rules live, so
|
| 361 |
# the policy was not actually cleared.
|
| 362 |
-
lambda v: ("", "", "*No policy loaded*", [new_rule()], (v or 0) + 1, ""
|
| 363 |
inputs=rules_version_state,
|
| 364 |
outputs=[current_policy_state, manual_text, policy_preview,
|
| 365 |
-
structured_rules_state, rules_version_state, policy_label_state,
|
| 366 |
-
policy_source_state],
|
| 367 |
)
|
| 368 |
|
| 369 |
# Sync UI components when state changes externally (e.g., from dataset load)
|
|
@@ -383,6 +382,4 @@ def build_policy_tab(base_dir: str, demo: gr.Blocks | None = None) -> dict:
|
|
| 383 |
# Exposed so restoring a saved policy can force the rule editor to re-render.
|
| 384 |
"rules_version_state": rules_version_state,
|
| 385 |
"policy_label_state": policy_label_state,
|
| 386 |
-
"policy_source_state": policy_source_state,
|
| 387 |
-
"effective_rules_state": effective_rules_state,
|
| 388 |
}
|
|
|
|
| 10 |
from utils.helpers import load_policy_from_file, load_preset_policy, list_preset_policy_names
|
| 11 |
from utils.policy_rules import (
|
| 12 |
new_rule,
|
| 13 |
+
policy_text_to_categories,
|
| 14 |
parse_policy_markdown,
|
| 15 |
rule_has_content,
|
| 16 |
rule_is_enabled,
|
|
|
|
| 40 |
# hand-written). The Testing tab shows this so nobody runs content against a
|
| 41 |
# policy they have forgotten they loaded.
|
| 42 |
policy_label_state = gr.State(value="")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 43 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 44 |
|
| 45 |
# --- Structured rule editor (model-agnostic; native where supported) -------------------------------
|
| 46 |
with gr.Accordion("🧩 Structured Rules (recommended — one score per rule)", open=True):
|
|
|
|
| 181 |
outputs=[structured_rules_state, rules_version_state],
|
| 182 |
)
|
| 183 |
|
| 184 |
+
def sync_rules_to_policy(rules, current_policy):
|
| 185 |
+
"""The rule list is the policy; render it for models that read text."""
|
| 186 |
+
if rules_have_content(rules):
|
| 187 |
return render_rules_to_text(rules)
|
| 188 |
return current_policy
|
| 189 |
|
|
|
|
|
|
|
|
|
|
|
|
|
| 190 |
# Live limit warning + keep the effective policy text in sync.
|
| 191 |
structured_rules_state.change(
|
| 192 |
policy_limits_markdown,
|
|
|
|
| 195 |
)
|
| 196 |
structured_rules_state.change(
|
| 197 |
sync_rules_to_policy,
|
| 198 |
+
inputs=[structured_rules_state, current_policy_state],
|
| 199 |
outputs=current_policy_state,
|
| 200 |
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 201 |
|
| 202 |
# Existing Policy Accordion
|
| 203 |
with gr.Accordion("📥 Load Existing Policy", open=False):
|
|
|
|
| 219 |
with gr.Accordion("✏️ Manual Editing", open=False):
|
| 220 |
manual_text = gr.Textbox(
|
| 221 |
label="Policy Text",
|
| 222 |
+
placeholder="Write or paste a policy, then add it as rules...",
|
| 223 |
lines=20
|
| 224 |
)
|
| 225 |
+
add_as_rules_btn = gr.Button("➕ Add as rules", variant="primary")
|
| 226 |
+
gr.Markdown(
|
| 227 |
+
"Turns whatever is in the box into rules and appends them to the list "
|
| 228 |
+
"above. A policy written as `## Name` with `Flag:` / `Don't flag:` / "
|
| 229 |
+
"`Exception:` lines becomes one rule per section; anything else becomes "
|
| 230 |
+
"a single rule you can then edit or delete."
|
| 231 |
+
)
|
| 232 |
|
| 233 |
policy_preview = gr.Markdown(value="*No policy loaded*")
|
| 234 |
|
|
|
|
| 244 |
both in the same list, each tagged with where it came from, so a rule you
|
| 245 |
added sits beside the preset's and can be compared, switched off or edited.
|
| 246 |
"""
|
| 247 |
+
blank = ("", "*No policy loaded*", "", [new_rule()], (version or 0) + 1, "")
|
| 248 |
if not name:
|
| 249 |
return blank
|
| 250 |
|
|
|
|
| 264 |
"enabled": True, "source": name}
|
| 265 |
for c in parsed
|
| 266 |
] or [new_rule()]
|
| 267 |
+
return policy_text, policy_text, policy_text, rules, (version or 0) + 1, name
|
| 268 |
|
| 269 |
load_preset_btn.click(
|
| 270 |
load_preset_handler,
|
| 271 |
inputs=[preset_dropdown, uploaded_policies_state, rules_version_state],
|
| 272 |
outputs=[current_policy_state, manual_text, policy_preview,
|
| 273 |
+
structured_rules_state, rules_version_state, policy_label_state],
|
|
|
|
| 274 |
)
|
| 275 |
|
| 276 |
def load_upload_handler(f, uploaded_policies):
|
|
|
|
| 315 |
load_upload_handler,
|
| 316 |
inputs=[upload_file, uploaded_policies_state],
|
| 317 |
outputs=[current_policy_state, manual_text, policy_preview, preset_dropdown, uploaded_policies_state],
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 318 |
)
|
| 319 |
|
| 320 |
+
def add_as_rules_handler(text, rules, version, label):
|
| 321 |
+
"""Append whatever is in the text box to the rule list.
|
| 322 |
+
|
| 323 |
+
One button for both routes: a preset already arrives as rules, and this
|
| 324 |
+
turns anything written or pasted into rules too, so the list is always the
|
| 325 |
+
policy and nothing is silently overridden by something typed elsewhere.
|
| 326 |
+
"""
|
| 327 |
+
if not (text or "").strip():
|
| 328 |
+
return rules, version, label, gr.update()
|
| 329 |
+
parsed = policy_text_to_categories(text)
|
| 330 |
+
added = [
|
| 331 |
+
{"name": c.get("name", ""), "flag": c.get("violation_rule", ""),
|
| 332 |
+
"dont_flag": c.get("not_violation_rule", ""),
|
| 333 |
+
"exception": c.get("exception_override", ""),
|
| 334 |
+
"context": c.get("context", ""), "enabled": True, "source": ""}
|
| 335 |
+
for c in parsed
|
| 336 |
+
]
|
| 337 |
+
kept = [r for r in (rules or []) if rule_has_content(r)]
|
| 338 |
+
return kept + added, (version or 0) + 1, (label or "Custom (hand-written)"), gr.update(value="")
|
| 339 |
+
|
| 340 |
def update_preview(text):
|
| 341 |
return text if text else "*No policy loaded*"
|
| 342 |
|
| 343 |
# Update state and preview when user leaves the textbox (blur event)
|
| 344 |
+
add_as_rules_btn.click(
|
| 345 |
+
add_as_rules_handler,
|
| 346 |
+
inputs=[manual_text, structured_rules_state, rules_version_state, policy_label_state],
|
| 347 |
+
outputs=[structured_rules_state, rules_version_state, policy_label_state, manual_text],
|
| 348 |
+
)
|
| 349 |
+
|
| 350 |
manual_text.blur(
|
| 351 |
+
# Keeps the preview in step while writing. The text only becomes the policy
|
| 352 |
+
# when "Add as rules" is pressed, so there is one route in and no silent
|
| 353 |
+
# override of the rule list.
|
| 354 |
+
lambda t: update_preview(t),
|
| 355 |
inputs=manual_text,
|
| 356 |
+
outputs=policy_preview,
|
| 357 |
)
|
| 358 |
|
| 359 |
clear_policy_btn.click(
|
| 360 |
# Clear the rules too: clearing only the text left loaded rules live, so
|
| 361 |
# the policy was not actually cleared.
|
| 362 |
+
lambda v: ("", "", "*No policy loaded*", [new_rule()], (v or 0) + 1, ""),
|
| 363 |
inputs=rules_version_state,
|
| 364 |
outputs=[current_policy_state, manual_text, policy_preview,
|
| 365 |
+
structured_rules_state, rules_version_state, policy_label_state],
|
|
|
|
| 366 |
)
|
| 367 |
|
| 368 |
# Sync UI components when state changes externally (e.g., from dataset load)
|
|
|
|
| 382 |
# Exposed so restoring a saved policy can force the rule editor to re-render.
|
| 383 |
"rules_version_state": rules_version_state,
|
| 384 |
"policy_label_state": policy_label_state,
|
|
|
|
|
|
|
| 385 |
}
|
|
@@ -25,7 +25,7 @@ import math
|
|
| 25 |
from utils.constants import POLICYLM_MAX_CATEGORIES, POLICYLM_MAX_POLICY_TOKENS
|
| 26 |
|
| 27 |
# The empty-rule template. Copy it (`dict(EMPTY_RULE)`) — never mutate the shared one.
|
| 28 |
-
EMPTY_RULE = {"name": "", "flag": "", "dont_flag": "", "exception": "", "enabled": True}
|
| 29 |
|
| 30 |
_FIELD_ORDER = ("name", "flag", "dont_flag", "exception")
|
| 31 |
|
|
@@ -91,6 +91,11 @@ def render_rules_to_text(rules: list[dict] | None) -> str:
|
|
| 91 |
"**Exception**. Evaluate each rule independently.",
|
| 92 |
"",
|
| 93 |
]
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 94 |
for i, rule in enumerate(kept, start=1):
|
| 95 |
name = _s(rule, "name") or f"Rule {i}"
|
| 96 |
lines.append(f"## {i}. {name}")
|
|
@@ -122,9 +127,10 @@ def rules_to_categories(rules: list[dict] | None) -> list[dict]:
|
|
| 122 |
if not flag:
|
| 123 |
# Without a violation rule there is nothing for PolicyLM to score.
|
| 124 |
continue
|
|
|
|
| 125 |
cat = {
|
| 126 |
"name": _s(rule, "name") or f"Rule {i}",
|
| 127 |
-
"violation_rule": flag,
|
| 128 |
}
|
| 129 |
dont = _s(rule, "dont_flag")
|
| 130 |
exc = _s(rule, "exception")
|
|
@@ -219,6 +225,44 @@ def _one_line(text: str) -> str:
|
|
| 219 |
return " ".join(str(text).split())
|
| 220 |
|
| 221 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 222 |
def parse_policy_markdown(policy_text: str, max_categories: int = POLICYLM_MAX_CATEGORIES) -> list[dict]:
|
| 223 |
"""Parse a written policy into PolicyLM categories.
|
| 224 |
|
|
@@ -227,6 +271,7 @@ def parse_policy_markdown(policy_text: str, max_categories: int = POLICYLM_MAX_C
|
|
| 227 |
violation rule is nothing for PolicyLM to score. Returns an empty list when the
|
| 228 |
document has no parseable sections, so the caller chooses the fallback.
|
| 229 |
"""
|
|
|
|
| 230 |
sections: list[tuple[str, list[str]]] = []
|
| 231 |
current: tuple[str, list[str]] | None = None
|
| 232 |
for line in (policy_text or "").splitlines():
|
|
@@ -252,7 +297,11 @@ def parse_policy_markdown(policy_text: str, max_categories: int = POLICYLM_MAX_C
|
|
| 252 |
if value:
|
| 253 |
found[field] = value
|
| 254 |
if "violation_rule" in found:
|
| 255 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 256 |
if len(categories) >= max_categories:
|
| 257 |
break
|
| 258 |
return categories
|
|
|
|
| 25 |
from utils.constants import POLICYLM_MAX_CATEGORIES, POLICYLM_MAX_POLICY_TOKENS
|
| 26 |
|
| 27 |
# The empty-rule template. Copy it (`dict(EMPTY_RULE)`) — never mutate the shared one.
|
| 28 |
+
EMPTY_RULE = {"name": "", "flag": "", "dont_flag": "", "exception": "", "enabled": True, "context": ""}
|
| 29 |
|
| 30 |
_FIELD_ORDER = ("name", "flag", "dont_flag", "exception")
|
| 31 |
|
|
|
|
| 91 |
"**Exception**. Evaluate each rule independently.",
|
| 92 |
"",
|
| 93 |
]
|
| 94 |
+
seen_context = list(dict.fromkeys(_s(r, "context") for r in kept if _s(r, "context")))
|
| 95 |
+
if seen_context:
|
| 96 |
+
# Once at the top for models that read the whole document; the per-rule copies
|
| 97 |
+
# exist only for models scored a category at a time.
|
| 98 |
+
lines += ["**Definitions and scope:** " + " ".join(seen_context), ""]
|
| 99 |
for i, rule in enumerate(kept, start=1):
|
| 100 |
name = _s(rule, "name") or f"Rule {i}"
|
| 101 |
lines.append(f"## {i}. {name}")
|
|
|
|
| 127 |
if not flag:
|
| 128 |
# Without a violation rule there is nothing for PolicyLM to score.
|
| 129 |
continue
|
| 130 |
+
context = _s(rule, "context")
|
| 131 |
cat = {
|
| 132 |
"name": _s(rule, "name") or f"Rule {i}",
|
| 133 |
+
"violation_rule": f"{flag} ({context})" if context else flag,
|
| 134 |
}
|
| 135 |
dont = _s(rule, "dont_flag")
|
| 136 |
exc = _s(rule, "exception")
|
|
|
|
| 225 |
return " ".join(str(text).split())
|
| 226 |
|
| 227 |
|
| 228 |
+
def policy_definitions(policy_text: str) -> list[tuple[str, str]]:
|
| 229 |
+
"""Definitions stated before the first "## " heading, as (term, line) pairs.
|
| 230 |
+
|
| 231 |
+
A policy states its scope once at the top — "Protected classes: …",
|
| 232 |
+
"Not protected: …". A model handed the whole document reads that; a model scored
|
| 233 |
+
one category at a time never sees it, which measurably changes verdicts.
|
| 234 |
+
|
| 235 |
+
Only lines of the form "Term: definition" count. Prose in the preamble is left for
|
| 236 |
+
the rendered document, since there is no term to attach it to.
|
| 237 |
+
"""
|
| 238 |
+
head = (policy_text or "").split("\n## ")[0]
|
| 239 |
+
out: list[tuple[str, str]] = []
|
| 240 |
+
for raw in head.splitlines():
|
| 241 |
+
line = raw.strip().lstrip("-* ").strip()
|
| 242 |
+
if line.startswith("#") or ":" not in line:
|
| 243 |
+
continue
|
| 244 |
+
term, _, rest = line.partition(":")
|
| 245 |
+
if term and rest.strip() and len(term.split()) <= 4:
|
| 246 |
+
out.append((_one_line(term).lower(), _one_line(line)))
|
| 247 |
+
return out
|
| 248 |
+
|
| 249 |
+
|
| 250 |
+
def _relevant_definitions(text: str, definitions: list[tuple[str, str]]) -> str:
|
| 251 |
+
"""The definitions this rule actually relies on.
|
| 252 |
+
|
| 253 |
+
Attached per rule rather than to all of them: repeating every definition on every
|
| 254 |
+
category burns the token budget and tells a model about terms its rule never uses.
|
| 255 |
+
A term matches on its longest word ("protected" for "Protected classes"), which
|
| 256 |
+
also catches the singular.
|
| 257 |
+
"""
|
| 258 |
+
lowered = (text or "").lower()
|
| 259 |
+
kept = [
|
| 260 |
+
line for term, line in definitions
|
| 261 |
+
if any(w for w in sorted(term.split(), key=len, reverse=True)[:1] if w in lowered)
|
| 262 |
+
]
|
| 263 |
+
return " ".join(dict.fromkeys(kept))
|
| 264 |
+
|
| 265 |
+
|
| 266 |
def parse_policy_markdown(policy_text: str, max_categories: int = POLICYLM_MAX_CATEGORIES) -> list[dict]:
|
| 267 |
"""Parse a written policy into PolicyLM categories.
|
| 268 |
|
|
|
|
| 271 |
violation rule is nothing for PolicyLM to score. Returns an empty list when the
|
| 272 |
document has no parseable sections, so the caller chooses the fallback.
|
| 273 |
"""
|
| 274 |
+
definitions = policy_definitions(policy_text)
|
| 275 |
sections: list[tuple[str, list[str]]] = []
|
| 276 |
current: tuple[str, list[str]] | None = None
|
| 277 |
for line in (policy_text or "").splitlines():
|
|
|
|
| 297 |
if value:
|
| 298 |
found[field] = value
|
| 299 |
if "violation_rule" in found:
|
| 300 |
+
entry = {"name": _one_line(name)[:120], **found}
|
| 301 |
+
context = _relevant_definitions(" ".join([name, *found.values()]), definitions)
|
| 302 |
+
if context:
|
| 303 |
+
entry["context"] = context
|
| 304 |
+
categories.append(entry)
|
| 305 |
if len(categories) >= max_categories:
|
| 306 |
break
|
| 307 |
return categories
|