Blame for scripts/lib/refresh.py

"""/refresh orchestrator. Faculty mode (this task) and maintainer mode (Task 27)."""
from __future__ import annotations

import json
import subprocess
from datetime import datetime, timezone
from pathlib import Path
from typing import Optional

from lib import git_ops, index_generator, local_state, source_state, sync
from lib.paths import repo_root


class IndexDriftError(RuntimeError):
    pass


class MissingStateError(RuntimeError):
    """Raised when STATE.json is missing — the maintainer hasn't run /refresh yet."""


class MissingIndexError(RuntimeError):
    """Raised when committed index/index.json is missing."""


_REFRESH_ALLOWLIST = [".me-hub", ".claude/settings.local.json"]


def _ensure_clean_worktree() -> None:
    if not git_ops.worktree_clean(allowlist=_REFRESH_ALLOWLIST):
        raise RuntimeError(
            "refresh requires a clean worktree (allowed exceptions: .me-hub/, "
            "settings.local.json). Resolve outstanding changes first."
        )


def run_faculty_mode() -> None:
    """git pull -> regenerate index to compare -> error on drift -> update local-state.

    Named invariant: this function MUST NOT mutate tracked files. It only writes
    to the gitignored .me-hub/local-state.json. On any drift between regenerated
    and committed index, it raises IndexDriftError without writing anything.
    """
    _ensure_clean_worktree()

    try:
        git_ops.pull()
    except subprocess.CalledProcessError:
        pass

    state = source_state.read()
    if state is None:
        raise MissingStateError(
            "STATE.json missing. Faculty mode expects a maintainer-produced STATE.json on main. "
            "Run /refresh on the maintainer machine first, then git pull on this machine."
        )

    sha = git_ops.current_commit()
    regenerated = index_generator.generate(source_commit=sha)

    committed_path = repo_root() / "index" / "index.json"
    if not committed_path.is_file():
        raise MissingIndexError("committed index/index.json is missing")
    committed = json.loads(committed_path.read_text(encoding="utf-8"))

    # Compare on content_fingerprint only — generated_utc and source_commit
    # differ legitimately across maintainer-write and faculty-regenerate.
    if regenerated["content_fingerprint"] != committed["content_fingerprint"]:
        raise IndexDriftError(
            f"regenerated index differs from committed index.\n"
            f"  regenerated content_fingerprint: {regenerated['content_fingerprint']}\n"
            f"  committed content_fingerprint:   {committed['content_fingerprint']}\n"
            "Investigate (out-of-date local fixtures or maintainer regression). "
            "Faculty mode never rewrites tracked files."
        )

    local_state.update_last_refresh(state_commit=state["source_commit"])
    print("OK faculty refresh: index matches committed; local-state updated.")


def run_maintainer_mode(
    *,
    box_togo_path: Path,
    tolk_syntopicon_path: Path,
    ferpa_lectures_raw: Optional[Path],
    allow_deletes: bool,
    push: bool,
) -> None:
    """Maintainer-mode refresh: per-source ingestion + regenerate + commit + push.

    Per-source allowlist commits: each `git add` is by exact path. We never use
    `git add .` or `git add -A`, so untracked files in other parts of the tree
    don't get accidentally committed.
    """
    _ensure_clean_worktree()
    try:
        git_ops.pull()
    except subprocess.CalledProcessError:
        pass

    repo = repo_root()

    _ingest_source(
        src=box_togo_path,
        tgt=repo / "content" / "togo",
        commit_paths=["content/togo"],
        commit_message="refresh: ingest togo content",
        allow_deletes=allow_deletes,
    )

    from lib import syntopicon_ingest
    syntopicon_ingest.ingest(
        tolk_syntopicon_path=tolk_syntopicon_path,
        target_dir=repo / "content" / "syntopicon-mgrecon780",
        course_pattern="MGRECON780_",
        expand_hops=1,
        allow_deletes=allow_deletes,
    )
    # syntopicon_ingest stamps `filter.ingested_utc` every run; restore the prior
    # _registry.json if the only change is that timestamp, so unchanged inputs
    # don't produce phantom diffs.
    _restore_file_if_only_timestamp_changed(
        repo / "content" / "syntopicon-mgrecon780" / "_registry.json",
        ("filter", "ingested_utc"),
    )
    git_ops.add_and_commit_if_changes(
        ["content/syntopicon-mgrecon780"],
        "refresh: ingest syntopicon (filtered to MGRECON780+1-hop)",
    )

    if ferpa_lectures_raw and ferpa_lectures_raw.is_dir():
        _normalize_lectures(ferpa_lectures_raw, repo / "content" / "lectures-jsonl" / "normalized")
        git_ops.add_and_commit_if_changes(
            ["content/lectures-jsonl/normalized"],
            "refresh: normalize lecture jsonl",
        )

    sha = git_ops.current_commit()
    idx = index_generator.generate(source_commit=sha)
    index_generator.write_index(idx)

    state = _build_state(idx=idx, source_commit=sha)
    source_state.write(state)

    git_ops.add_and_commit(
        ["index/index.json", "STATE.json"],
        "refresh: regenerate index",
    )

    # Auto-generate topic-to-syntopicon-term mapping
    from lib import topic_links
    links_dir = repo / "index" / "inputs" / "syntopicon-links"
    topic_links.run(
        index_path=repo / "index" / "index.json",
        registry_path=repo / "content" / "syntopicon-mgrecon780" / "_registry.json",
        output_dir=links_dir,
    )
    # topic_links.run() stamps `generated_utc` into both output files every call;
    # restore the prior content if the only change is that timestamp.
    _restore_file_if_only_timestamp_changed(links_dir / "topic-to-entry.json", ("generated_utc",))
    _restore_file_if_only_timestamp_changed(links_dir / "_unresolved.json", ("generated_utc",))
    git_ops.add_and_commit_if_changes(
        ["index/inputs/syntopicon-links"],
        "refresh: regenerate topic-to-entry map",
    )

    if push:
        git_ops.push()

    print("OK maintainer refresh complete.")


def _ingest_source(
    *,
    src: Path,
    tgt: Path,
    commit_paths: list[str],
    commit_message: str,
    allow_deletes: bool,
) -> None:
    if not src.is_dir():
        raise RuntimeError(f"source missing: {src}. Path required in maintainer mode.")

    tgt.mkdir(parents=True, exist_ok=True)
    diff = sync.stage_and_diff(src, tgt)
    if not (diff.additions or diff.modifications or diff.deletions):
        return

    print(f"  staged sync of {src} -> {tgt.relative_to(repo_root())}:")
    print(sync.format_diff_summary(diff))
    sync.apply(diff, src, tgt, allow_deletes=allow_deletes)

    git_ops.add_and_commit(commit_paths, commit_message)


def _restore_file_if_only_timestamp_changed(
    file_path: Path, timestamp_field_path: tuple[str, ...]
) -> None:
    """If file_path differs from the HEAD version ONLY in the named timestamp
    field, restore the HEAD content. Keeps maintainer-mode refresh idempotent on
    unchanged upstream input (which would otherwise re-commit every run because
    the generator stamps the timestamp on every write).

    `timestamp_field_path` is a sequence of dict keys naming the location of the
    timestamp inside the JSON document. Examples:
        ("generated_utc",)              # top-level field
        ("filter", "ingested_utc")      # nested under "filter"

    No-op if the file isn't tracked yet (first ingest), JSON parsing fails, or
    the structural content (everything except the named timestamp) actually
    differs.
    """
    if not file_path.is_file():
        return

    repo = repo_root()
    try:
        rel_path = file_path.resolve().relative_to(repo.resolve())
    except ValueError:
        # Path isn't under the repo root; we can't address it via git.
        return
    rel_posix = rel_path.as_posix()

    try:
        head_blob = subprocess.run(
            ["git", "show", f"HEAD:{rel_posix}"],
            check=True, capture_output=True, text=True,
        ).stdout
    except subprocess.CalledProcessError:
        # Not tracked yet; nothing to restore.
        return

    try:
        current = json.loads(file_path.read_text(encoding="utf-8"))
        head = json.loads(head_blob)
    except json.JSONDecodeError:
        return

    def _stripped(doc: dict) -> dict:
        # Walk into nested dicts following timestamp_field_path[:-1], pop the
        # leaf key. Returns a structurally-equal copy with the timestamp removed.
        if not timestamp_field_path:
            return doc
        out = dict(doc)
        cursor = out
        for key in timestamp_field_path[:-1]:
            inner = cursor.get(key)
            if not isinstance(inner, dict):
                return out
            cursor[key] = dict(inner)
            cursor = cursor[key]
        cursor.pop(timestamp_field_path[-1], None)
        return out

    if _stripped(current) == _stripped(head):
        file_path.write_text(head_blob, encoding="utf-8")


def _normalize_lectures(ferpa_raw_dir: Path, target_dir: Path) -> None:
    """Apply redaction policy to every raw JSONL in ferpa_raw_dir.

    Each input file is scrubbed via redaction.scrub_to_instructor_only(), which
    drops Anonymous Student turns, redacts known student names from instructor
    text and problem outcomes, and writes the redacted JSONL atomically.

    The .redaction-report.txt files contain absolute source/output paths
    (machine-specific, not deterministic across maintainers) and would trip
    index_generator's unmatched-files diagnostic if left under content/. We
    pass an explicit report_path under .me-hub/redaction-reports/ (gitignored)
    so the report is written directly there, avoiding the non-atomic
    scrub-then-move window where a SIGKILL could leave a stray report under
    content/.

    No source_hash short-circuit: scrub_to_instructor_only is deterministic
    given identical input + config, so the rewritten JSONL is byte-identical
    on repeat runs and add_and_commit_if_changes handles the no-op case.
    """
    from lib import redaction
    cfg = redaction.default_config()
    target_dir.mkdir(parents=True, exist_ok=True)

    # Defensive: clean up any pre-existing stray report files (in case a prior
    # version wrote them here). index_generator would otherwise treat them as
    # unmatched.
    for stray in target_dir.glob("*.redaction-report.txt"):
        stray.unlink()

    reports_dir = repo_root() / ".me-hub" / "redaction-reports"
    reports_dir.mkdir(parents=True, exist_ok=True)

    for raw_jsonl in sorted(ferpa_raw_dir.glob("*.jsonl")):
        out = target_dir / raw_jsonl.name
        report = reports_dir / (raw_jsonl.stem + ".redaction-report.txt")
        redaction.scrub_to_instructor_only(raw_jsonl, out, cfg, report_path=report)


def _build_state(idx: dict, source_commit: str) -> dict:
    repo = repo_root()
    # Ensure all source directories exist for fingerprinting; empty dir → empty-input digest.
    (repo / "content" / "togo").mkdir(parents=True, exist_ok=True)
    (repo / "content" / "lectures-jsonl" / "normalized").mkdir(parents=True, exist_ok=True)
    (repo / "content" / "syntopicon-mgrecon780").mkdir(parents=True, exist_ok=True)
    now_iso = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
    return {
        "schema_version": 1,
        "snapshot_utc": now_iso,
        "source_commit": source_commit,
        "sources": {
            "togo": {
                "ingested_utc": now_iso,
                "file_count": _count_files(repo / "content" / "togo"),
                "content_fingerprint": source_state.compute_directory_fingerprint(
                    repo / "content" / "togo"
                ),
            },
            "lectures_jsonl": {
                "ingested_utc": now_iso,
                "file_count": _count_files(repo / "content" / "lectures-jsonl" / "normalized"),
                "content_fingerprint": source_state.compute_directory_fingerprint(
                    repo / "content" / "lectures-jsonl" / "normalized"
                ),
                "redaction_policy": "v1.0",
            },
            "syntopicon": {
                "ingested_utc": now_iso,
                "entry_count": _count_files(repo / "content" / "syntopicon-mgrecon780"),
                "content_fingerprint": source_state.compute_directory_fingerprint(
                    repo / "content" / "syntopicon-mgrecon780"
                ),
            },
        },
        "index": {
            "generator_version": idx["generator_version"],
            "topics": len(idx["topics"]),
            "files_indexed": sum(len(t.get("files", [])) for t in idx["topics"].values()),
            "content_fingerprint": idx["content_fingerprint"],
        },
    }


def _count_files(d: Path) -> int:
    if not d.is_dir():
        return 0
    return sum(1 for p in d.rglob("*") if p.is_file() and p.name != ".gitkeep")