Spaces:
Running
Running
File size: 6,271 Bytes
526950e | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 | """Lecture JSONL redaction: instructor-only policy.
Drops all turns where speaker == "Anonymous Student". Redacts known student
first/last names from instructor turns and problem_outcome records. Whitelist
phrases preserved verbatim. Defaults loaded from config/redaction.yaml.
Atomic write via temp file + os.replace. Emits sibling redaction-report.txt.
"""
from __future__ import annotations
import json
import os
import re
from dataclasses import dataclass, field
from pathlib import Path
import yaml
REDACTION_POLICY_VERSION = "1.0"
_CONFIG_PATH = Path(__file__).parent.parent.parent / "config" / "redaction.yaml"
@dataclass
class RedactionConfig:
first_names: set = field(default_factory=set)
last_names: set = field(default_factory=set)
whitelist_phrases: list = field(default_factory=list)
replacement_token: str = "[STUDENT]"
last_name_replacement_token: str = "[REDACTED-LNAME]"
policy_version: str = REDACTION_POLICY_VERSION
@dataclass
class RedactionStats:
student_turns_dropped: int
instructor_turns_kept: int
session_headers_kept: int
problem_outcomes_kept: int
name_redactions_in_instructor: int
name_redactions_in_problem_outcomes: int
def default_config() -> RedactionConfig:
raw = yaml.safe_load(_CONFIG_PATH.read_text(encoding="utf-8"))
first = set(raw.get("verified_first_names", [])) | set(raw.get("defensive_first_names", []))
return RedactionConfig(
first_names=first,
last_names=set(raw.get("last_names", [])),
whitelist_phrases=list(raw.get("whitelist_phrases", [])),
replacement_token=raw.get("replacement_token", "[STUDENT]"),
last_name_replacement_token=raw.get("last_name_replacement_token", "[REDACTED-LNAME]"),
policy_version=raw.get("policy_version", REDACTION_POLICY_VERSION),
)
def _build_regexes(config):
whitelist_rx = re.compile("|".join(re.escape(p) for p in config.whitelist_phrases)) if config.whitelist_phrases else re.compile(r"$^")
name_rx = re.compile(
r"\b(" + "|".join(re.escape(n) for n in sorted(config.first_names, key=len, reverse=True)) + r")\b",
flags=re.IGNORECASE,
) if config.first_names else re.compile(r"$^")
last_rx = re.compile(
r"\b(" + "|".join(re.escape(n) for n in config.last_names) + r")\b",
flags=re.IGNORECASE,
) if config.last_names else re.compile(r"$^")
return whitelist_rx, name_rx, last_rx
def _redact_text(text, config, regexes):
whitelist_rx, name_rx, last_rx = regexes
placeholders = {}
def stash(m):
ph = f"<<WL{len(placeholders)}>>"
placeholders[ph] = m.group(0)
return ph
protected = whitelist_rx.sub(stash, text)
n = 0
def replace_first(m):
nonlocal n
n += 1
return config.replacement_token
redacted = name_rx.sub(replace_first, protected)
def replace_last(m):
nonlocal n
n += 1
return config.last_name_replacement_token
redacted = last_rx.sub(replace_last, redacted)
for ph, orig in placeholders.items():
redacted = redacted.replace(ph, orig)
return redacted, n
def scrub_to_instructor_only(
raw_path, out_path, config, *, report_path: Path | None = None
) -> RedactionStats:
regexes = _build_regexes(config)
student_dropped = 0
instructor_kept = 0
headers_kept = 0
outcomes_kept = 0
instr_redactions = 0
outcome_redactions = 0
out_path.parent.mkdir(parents=True, exist_ok=True)
tmp = out_path.with_suffix(out_path.suffix + ".tmp")
with open(raw_path, encoding="utf-8") as src, open(tmp, "w", encoding="utf-8") as dst:
for line in src:
line = line.strip()
if not line:
continue
try:
rec = json.loads(line)
except json.JSONDecodeError:
continue
rt = rec.get("record_type", "")
if rt == "session_header":
headers_kept += 1
dst.write(json.dumps(rec, ensure_ascii=False) + "\n")
elif rt == "turn":
speaker = rec.get("speaker", "")
if speaker == "Anonymous Student":
student_dropped += 1
continue
text, n = _redact_text(rec.get("text", "") or "", config, regexes)
rec["text"] = text
if n > 0:
rec["_redactions"] = n
instr_redactions += n
instructor_kept += 1
dst.write(json.dumps(rec, ensure_ascii=False) + "\n")
elif rt == "problem_outcome":
blob = json.dumps(rec, ensure_ascii=False)
redacted_blob, n = _redact_text(blob, config, regexes)
if n > 0:
rec = json.loads(redacted_blob)
rec["_redactions"] = n
outcome_redactions += n
outcomes_kept += 1
dst.write(json.dumps(rec, ensure_ascii=False) + "\n")
os.replace(tmp, out_path)
if report_path is None:
report_path = out_path.with_name(out_path.stem + ".redaction-report.txt")
report_body = (
f"REDACTION REPORT (policy v{config.policy_version})\n"
+ "=" * 50 + "\n"
+ f"Source: {raw_path}\nOutput: {out_path}\n\n"
+ f"Dropped student turns: {student_dropped}\n"
+ f"Kept instructor turns: {instructor_kept}\n"
+ f"Kept session headers: {headers_kept}\n"
+ f"Kept problem outcomes: {outcomes_kept}\n"
+ f"Redactions in instructor turns: {instr_redactions}\n"
+ f"Redactions in problem outcomes: {outcome_redactions}\n"
)
report_path.parent.mkdir(parents=True, exist_ok=True)
report_tmp = report_path.with_suffix(report_path.suffix + ".tmp")
report_tmp.write_text(report_body, encoding="utf-8")
os.replace(report_tmp, report_path)
return RedactionStats(
student_turns_dropped=student_dropped,
instructor_turns_kept=instructor_kept,
session_headers_kept=headers_kept,
problem_outcomes_kept=outcomes_kept,
name_redactions_in_instructor=instr_redactions,
name_redactions_in_problem_outcomes=outcome_redactions,
)
|