api/config_editor.py

"""Reads and writes the same two files a developer would otherwise hand-edit:
config/pii_categories.yaml and config/org_exclude_list.txt. This is what
lets the settings page in the frontend add a regex category, a
structural-table-column rule, or an exclude-list entry WITHOUT anyone
touching YAML or Python -- the whole point of this module is to be the one
place that knows how to safely rewrite those files.

Uses ruamel.yaml (round-trip mode), not plain PyYAML, specifically so the
explanatory comments already in pii_categories.yaml survive being edited
through the API -- plain PyYAML's dump() would silently delete every
comment in the file on the first save.
"""

from pathlib import Path

from ruamel.yaml import YAML

CONFIG_DIR = Path(__file__).resolve().parent.parent / "config"
CATEGORIES_FILE = CONFIG_DIR / "pii_categories.yaml"
EXCLUDE_LIST_FILE = CONFIG_DIR / "org_exclude_list.txt"

def _make_yaml() -> YAML:
    # A fresh instance per load/dump, deliberately -- ruamel.yaml's YAML
    # object accumulates internal composer/anchor state across reuse, and
    # sharing one instance across this module's many load() and dump()
    # calls (one per HTTP request, interleaved) intermittently corrupted
    # that state (`AttributeError: 'NoneType' object has no attribute
    # 'anchor'` from ruamel's composer). Instances are cheap; this is not
    # a hot path.
    yaml = YAML()
    yaml.preserve_quotes = True
    yaml.indent(mapping=2, sequence=4, offset=2)
    return yaml


# Only these two category "kind"s can be created through the API without
# writing code: both are pure data (a regex, or a column-header rule) with
# no logic of their own. ORGANIZATION/PHYSICAL_ADDRESS/DATE_OF_BIRTH/etc.
# (kind: spacy_org / structural / context_regex) are backed by specific
# functions in redactor/special_categories.py, so adding a *new* one of
# those genuinely does need a code change -- seeREADME's "Customizing"
# section for why that line is drawn there.
API_EDITABLE_KINDS = {"presidio_builtin", "regex"}


def _load_yaml() -> dict:
    with open(CATEGORIES_FILE, encoding="utf-8") as f:
        return _make_yaml().load(f)


def _save_yaml(data: dict):
    # newline="\n": without it, Python's text-mode write translates to the
    # OS line ending (CRLF on Windows), which would otherwise turn every
    # settings-UI edit into a full-file diff of line-ending churn.
    with open(CATEGORIES_FILE, "w", encoding="utf-8", newline="\n") as f:
        _make_yaml().dump(data, f)


# ---------------------------------------------------------------- categories

def list_categories() -> list[dict]:
    data = _load_yaml()
    return [dict(c) for c in data["categories"]]


def add_regex_category(name: str, patterns: list[str], score_threshold: float, fake_provider: str):
    data = _load_yaml()
    if any(c["name"] == name for c in data["categories"]):
        raise ValueError(f"A category named {name!r} already exists.")
    data["categories"].append({
        "name": name,
        "kind": "regex",
        "score_threshold": score_threshold,
        "patterns": list(patterns),
        "fake_provider": fake_provider,
    })
    _save_yaml(data)


def remove_category(name: str):
    data = _load_yaml()
    category = next((c for c in data["categories"] if c["name"] == name), None)
    if category is None:
        raise ValueError(f"No category named {name!r}.")
    if category["kind"] not in API_EDITABLE_KINDS:
        raise ValueError(
            f"{name!r} is a {category['kind']!r} category, backed by code in "
            "redactor/special_categories.py -- remove its block from "
            "pii_categories.yaml directly if you really want to disable it."
        )
    data["categories"].remove(category)
    _save_yaml(data)


# ------------------------------------------------------ structural columns

def list_structural_columns() -> list[dict]:
    data = _load_yaml()
    return [dict(rule) for rule in data.get("structural_table_columns", [])]


def add_structural_column(column_header: str, category: str, requires_any_of: list[str]):
    data = _load_yaml()
    data.setdefault("structural_table_columns", [])
    data["structural_table_columns"].append({
        "column_header": column_header.strip().lower(),
        "category": category,
        "requires_any_of": [r.strip().lower() for r in requires_any_of],
    })
    _save_yaml(data)


def remove_structural_column(index: int):
    data = _load_yaml()
    rules = data.get("structural_table_columns", [])
    if not (0 <= index < len(rules)):
        raise ValueError(f"No structural column rule at index {index}.")
    del rules[index]
    _save_yaml(data)


# -------------------------------------------------------------- deny-list
# config/org_exclude_list.txt is plain text (one term per line, '#'
# comments allowed) -- no YAML structure to preserve, so this is simpler
# than the functions above.

def list_exclude_terms() -> list[str]:
    lines = EXCLUDE_LIST_FILE.read_text(encoding="utf-8").splitlines()
    return [line.strip() for line in lines if line.strip() and not line.strip().startswith("#")]


def add_exclude_term(term: str):
    term = term.strip()
    if not term:
        raise ValueError("Term can't be empty.")
    if term.lower() in {t.lower() for t in list_exclude_terms()}:
        raise ValueError(f"{term!r} is already on the list.")
    with open(EXCLUDE_LIST_FILE, "a", encoding="utf-8", newline="\n") as f:
        f.write(f"{term}\n")


def remove_exclude_term(term: str):
    lines = EXCLUDE_LIST_FILE.read_text(encoding="utf-8").splitlines()
    kept = [line for line in lines if line.strip().lower() != term.strip().lower()]
    if len(kept) == len(lines):
        raise ValueError(f"{term!r} isn't on the list.")
    EXCLUDE_LIST_FILE.write_text("\n".join(kept) + "\n", encoding="utf-8", newline="\n")