Blame for scripts/lib/chat_eval/cli.py

"""argparse entry: me-hub feedback subcommands.

Invoke as: python -m scripts.lib.chat_eval.cli <subcommand> [...]
"""
from __future__ import annotations
import argparse
import json
import os
import sys
from datetime import datetime, timezone, timedelta
from pathlib import Path

from scripts.lib.chat_eval import analytics
from scripts.lib.chat_eval.report import build_report
from scripts.lib.chat_eval.state_io import fetch_state, push_state
from scripts.lib.runtime_config import DERIVED_STUDENT_CHAT_DIR, TURN_LOG_PATH


_DEFAULT_LOG = TURN_LOG_PATH


def _print_table(headers: list[str], rows: list[list[str]]) -> None:
    if not rows:
        print("(no rows)")
        return
    print("| " + " | ".join(headers) + " |")
    print("|" + "|".join("---" for _ in headers) + "|")
    for row in rows:
        print("| " + " | ".join(str(c) for c in row) + " |")


def _cmd_report(args) -> int:
    md = build_report(args.log_path, since=args.since)
    if args.out and args.out != "-":
        Path(args.out).write_text(md, encoding="utf-8")
        print(f"Report written to {args.out}")
    else:
        print(md)
    return 0


def _cmd_coverage(args) -> int:
    result = analytics.coverage(args.log_path, since=args.since)
    if args.json:
        print(json.dumps(result, indent=2))
        return 0
    rows = sorted(result.items(), key=lambda kv: -kv[1])
    _print_table(["term", "count"], [[k, v] for k, v in rows])
    return 0


def _cmd_misconceptions(args) -> int:
    result = analytics.misconception_frequency(args.log_path, since=args.since)
    if args.json:
        print(json.dumps(result, indent=2))
        return 0
    rows = sorted(result.items(), key=lambda kv: -kv[1]["count"])
    _print_table(
        ["id", "count", "handled"],
        [[k, v["count"], v["handled"]] for k, v in rows],
    )
    return 0


def _cmd_doc_utilization(args) -> int:
    result = analytics.doc_utilization(args.log_path, since=args.since)
    if args.json:
        print(json.dumps(result, indent=2))
        return 0
    rows = sorted(result.items(),
                  key=lambda kv: (kv[1]["util_pct"], -kv[1]["retrieved_count"]))
    _print_table(
        ["doc_id", "retrieved", "used", "util %"],
        [[k, v["retrieved_count"], v["used_count"], v["util_pct"]] for k, v in rows],
    )
    return 0


def _cmd_voice_failures(args) -> int:
    result = analytics.voice_failures(args.log_path, since=args.since)
    if args.json:
        print(json.dumps(result, indent=2))
        return 0
    if not result:
        print("(no voice failures)")
        return 0
    for v in result:
        print(f"### {v['turn_id']} (voice {v['voice_score']})")
        print(f"- Query: {v['user_message']}")
        print(f"- Response: {v['response'][:200]}")
        print(f"- Diagnosis: {v['diagnosis']}")
        print()
    return 0


def _cmd_timing(args) -> int:
    result = analytics.timing_density(args.log_path, since=args.since)
    if args.json:
        print(json.dumps(result, indent=2))
        return 0
    print("| weekday | total |")
    print("|---|---|")
    for wd in ["Mon", "Tue", "Wed", "Thu", "Fri", "Sat", "Sun"]:
        print(f"| {wd} | {result.get(wd, {}).get('total', 0)} |")
    return 0


def _cmd_judge(args) -> int:
    """Run the judge over unjudged turns. Real implementation requires an
    Anthropic client and the catalog; assemble both and dispatch."""
    from scripts.lib.catalog_loader import load_catalog
    from scripts.lib.chat_eval.judge import judge_turn, JudgeError, JudgeTransientError
    from scripts.lib.chat_eval.turn_log import (
        is_student_mode_turn, iter_turns, update_judge,
    )

    catalog = load_catalog(DERIVED_STUDENT_CHAT_DIR)
    turns = list(iter_turns(args.log_path, since=args.since))
    unjudged = [
        t for t in turns
        if t.get("judge") is None and is_student_mode_turn(t)
    ]
    if args.limit:
        unjudged = unjudged[: args.limit]
    if not unjudged:
        print("(no unjudged turns)")
        return 0

    print(f"Judging {len(unjudged)} turns...")
    terminal = 0
    transient = 0
    for i, turn in enumerate(unjudged, 1):
        print(f"[{i}/{len(unjudged)}] {turn['turn_id']}")
        try:
            judge_dict = judge_turn(turn, catalog)
            update_judge(turn["turn_id"], judge_dict, args.log_path)
        except JudgeTransientError as e:
            # The turn stays unjudged; tomorrow's run will retry it.
            print(f"  SKIPPED (transient): {e}")
            transient += 1
        except JudgeError as e:
            print(f"  FAILED: {e}")
            terminal += 1
    judged = len(unjudged) - terminal - transient
    print(
        f"Done. {judged} judged, {transient} skipped (transient: will retry "
        f"next run), {terminal} failed (terminal)."
    )
    return 0 if terminal == 0 else 1


def _cmd_notes(args) -> int:
    """Run the notes summarizer for each faculty login."""
    from pathlib import Path as _P
    from scripts.lib.chat_eval import notes_io, notes_summarizer, tuning
    from scripts.lib.chat.prompt_builder import FACULTY_LOGINS

    missing = _missing_hf_env()
    if missing:
        print(f"ERROR: missing required env var(s): {', '.join(missing)}",
              file=sys.stderr)
        return 2

    repo_id = os.environ["ME_CHAT_HISTORY_DATASET_REPO_ID"]
    token = os.environ["HF_TOKEN"]
    derived_dir = args.derived_dir
    turns_path = derived_dir / "turns.jsonl"
    archive_dir = derived_dir / "notes" / "archive"
    archive_dir.mkdir(parents=True, exist_ok=True)

    prior_rows = notes_io.fetch_summarizer_log(repo_id=repo_id, token=token)

    now_ts = tuning.now_utc_iso()
    llm = notes_summarizer.make_anthropic_client()

    overall_rows = list(prior_rows)
    had_terminal = False

    for login in sorted(FACULTY_LOGINS):
        print(f"[{login}] running summarizer...")
        current_body = notes_io.fetch_user_notes(
            login=login, repo_id=repo_id, token=token,
        )
        try:
            new_body, audit = notes_summarizer.run_for_user(
                login,
                turns_path=turns_path,
                current_notes_body=current_body,
                prior_log_rows=overall_rows,
                llm=llm,
                now_ts=now_ts,
            )
        except notes_summarizer.SummarizerTerminalError as e:
            print(f"  TERMINAL: {e}", file=sys.stderr)
            had_terminal = True
            overall_rows.append({
                "ts": now_ts, "login": login, "outcome": "terminal_failure",
                "since_ts": None, "last_processed_ts": None,
                "turns_processed": 0,
                "notes_chars_before": len(current_body),
                "notes_chars_after": None, "archived_to": None,
                "haiku_model": getattr(llm, "model_name", None),
                "elapsed_s": 0.0, "failure_reason": str(e),
            })
            continue

        if new_body is None:
            print(f"  outcome={audit['outcome']} (no notes write)")
            overall_rows.append(audit)
            continue

        archive_local = notes_io.next_archive_path_local(
            archive_dir=archive_dir, login=login, ts=now_ts,
        )
        archive_local.write_text(current_body, encoding="utf-8")
        archive_repo_path = notes_io.archive_path_for(login=login, ts=now_ts)
        if archive_local.name != _P(archive_repo_path).name:
            # Collision case: archive_path_for ignores suffixes; rebuild
            # the repo path to match the local -NN suffix.
            archive_repo_path = f"notes/archive/{archive_local.name}"
        audit["archived_to"] = archive_repo_path
        overall_rows.append(audit)

        notes_io.push_notes_update(
            repo_id=repo_id, token=token, login=login,
            new_notes_body=new_body,
            archive_repo_path=archive_repo_path,
            archive_body=current_body,
            summarizer_log_rows=overall_rows,
            commit_message=f"notes: {login} {now_ts[:10]}",
        )
        print(f"  outcome=updated (chars {audit['notes_chars_before']} → {audit['notes_chars_after']})")

    # If no successful update happened, still flush the log so no-op /
    # transient rows reach the dataset.
    if overall_rows != prior_rows:
        notes_io.push_summarizer_log_only(
            repo_id=repo_id, token=token,
            summarizer_log_rows=overall_rows,
            commit_message=f"notes: {now_ts[:10]} audit",
        )

    return 1 if had_terminal else 0


def _cmd_tune(args) -> int:
    from scripts.lib.chat_eval.tune_runner import run
    return run(args)


def _missing_hf_env() -> list[str]:
    """Return the list of HF Dataset env vars that are unset (empty list if all present)."""
    missing = []
    if not os.environ.get("ME_CHAT_HISTORY_DATASET_REPO_ID"):
        missing.append("ME_CHAT_HISTORY_DATASET_REPO_ID")
    if not os.environ.get("HF_TOKEN"):
        missing.append("HF_TOKEN")
    return missing


def _cmd_fetch_state(args) -> int:
    missing = _missing_hf_env()
    if missing:
        print(f"ERROR: missing required env var(s): {', '.join(missing)}",
              file=sys.stderr)
        return 2
    fetch_state(repo_id=os.environ["ME_CHAT_HISTORY_DATASET_REPO_ID"],
                token=os.environ["HF_TOKEN"],
                derived_dir=args.derived_dir)
    return 0


def _cmd_push_state(args) -> int:
    missing = _missing_hf_env()
    if missing:
        print(f"ERROR: missing required env var(s): {', '.join(missing)}",
              file=sys.stderr)
        return 2
    push_state(repo_id=os.environ["ME_CHAT_HISTORY_DATASET_REPO_ID"],
               token=os.environ["HF_TOKEN"],
               derived_dir=args.derived_dir,
               commit_message=args.commit_message)
    return 0


def _cmd_digest(args) -> int:
    """Build and optionally send the weekly HTML digest."""
    from scripts.lib.chat_eval import tuning
    from scripts.lib.chat_eval.digest import (
        faculty_digest,
        render_html,
        send_via_resend,
    )

    now_ts = tuning.now_utc_iso()
    now_dt = datetime.strptime(now_ts, "%Y-%m-%dT%H:%M:%SZ").replace(tzinfo=timezone.utc)
    since_dt = now_dt - timedelta(days=args.window_days)
    since_iso = since_dt.strftime("%Y-%m-%dT%H:%M:%SZ")
    today = now_dt.date().isoformat()

    body_md = faculty_digest(
        args.turns_log,
        args.tuning_log,
        since=since_iso,
        today=today,
        window_days=args.window_days,
    )
    html = render_html(body_md)

    if args.dry_run:
        sys.stdout.write(html)
        if not html.endswith("\n"):
            sys.stdout.write("\n")
        return 0

    api_key = os.environ.get("RESEND_API_KEY")
    if not api_key:
        print("ERROR: RESEND_API_KEY not set", file=sys.stderr)
        return 2

    recipients = [r.strip() for r in (args.to or "").split(",") if r.strip()]
    if not recipients:
        print("ERROR: --to must contain at least one email address",
              file=sys.stderr)
        return 2

    sender = args.sender or "onboarding@resend.dev"
    subject = f"MGRECON780 chat tutor — weekly briefing {today}"
    send_via_resend(
        api_key=api_key,
        sender=sender,
        recipients=recipients,
        subject=subject,
        html_body=html,
    )
    print(f"Sent digest to {len(recipients)} recipient(s).")
    return 0


def _build_parser() -> argparse.ArgumentParser:
    p = argparse.ArgumentParser(prog="python -m scripts.lib.chat_eval.cli")
    sub = p.add_subparsers(dest="command", required=True)

    def _common(parser):
        parser.add_argument("--log-path", type=Path, default=_DEFAULT_LOG,
                            help="Path to turns.jsonl (default: derived/student-chat/turns.jsonl)")
        parser.add_argument("--since", type=str, default=None,
                            help="ISO date filter (YYYY-MM-DD)")
        parser.add_argument("--json", action="store_true",
                            help="Output JSON instead of markdown table")

    p_report = sub.add_parser("report", help="Generate unified weekly report")
    _common(p_report)
    p_report.add_argument("--out", type=str, default="-",
                          help="Output path (default: stdout)")
    p_report.set_defaults(func=_cmd_report)

    p_coverage = sub.add_parser("coverage", help="Concept coverage")
    _common(p_coverage)
    p_coverage.set_defaults(func=_cmd_coverage)

    p_mc = sub.add_parser("misconceptions", help="Misconception frequency")
    _common(p_mc)
    p_mc.set_defaults(func=_cmd_misconceptions)

    p_du = sub.add_parser("doc-utilization", help="Per-doc utilization")
    _common(p_du)
    p_du.set_defaults(func=_cmd_doc_utilization)

    p_vf = sub.add_parser("voice-failures", help="Turns with low voice scores")
    _common(p_vf)
    p_vf.set_defaults(func=_cmd_voice_failures)

    p_t = sub.add_parser("timing", help="Weekday/hour density")
    _common(p_t)
    p_t.set_defaults(func=_cmd_timing)

    p_judge = sub.add_parser("judge", help="Run judge over unjudged turns")
    _common(p_judge)
    p_judge.add_argument("--limit", type=int, default=None,
                         help="Max turns to judge in this run")
    p_judge.set_defaults(func=_cmd_judge)

    p_tune = sub.add_parser("tune",
                            help="Propose one tuning step")
    p_tune.add_argument("--log-path", type=Path, default=_DEFAULT_LOG,
                        help="Path to turns.jsonl")
    p_tune.add_argument("--tuning-state-path", type=Path,
                        default=_DEFAULT_LOG.parent / "retrieval_tuning.json")
    p_tune.add_argument("--tuning-log-path", type=Path,
                        default=_DEFAULT_LOG.parent / "tuning_log.jsonl")
    p_tune.add_argument("--unfreeze", action="store_true",
                        help="Clear frozen flag; do not propose a step.")
    p_tune.add_argument("--dry-run", action="store_true",
                        help="Compute the step but do not write state.")
    p_tune.set_defaults(func=_cmd_tune)

    p_fs = sub.add_parser("fetch-state",
                          help="Download tuning state files from HF Dataset")
    p_fs.add_argument("--derived-dir", type=Path,
                      default=_DEFAULT_LOG.parent)
    p_fs.set_defaults(func=_cmd_fetch_state)

    p_ps = sub.add_parser("push-state",
                          help="Upload tuning state files to HF Dataset")
    p_ps.add_argument("--derived-dir", type=Path,
                      default=_DEFAULT_LOG.parent)
    p_ps.add_argument("--commit-message", type=str,
                      default="tuner: update state")
    p_ps.set_defaults(func=_cmd_push_state)

    p_notes = sub.add_parser(
        "notes",
        help="Run the per-faculty notes summarizer",
    )
    p_notes.add_argument("--derived-dir", type=Path,
                         default=_DEFAULT_LOG.parent,
                         help="Working dir for turns.jsonl + notes/ (default: derived/student-chat)")
    p_notes.set_defaults(func=_cmd_notes)

    p_digest = sub.add_parser(
        "digest",
        help="Build the weekly HTML digest and send via Resend",
    )
    p_digest.add_argument("--turns-log", type=Path, default=_DEFAULT_LOG,
                          help="Path to turns.jsonl")
    p_digest.add_argument("--tuning-log", type=Path,
                          default=_DEFAULT_LOG.parent / "tuning_log.jsonl",
                          help="Path to tuning_log.jsonl")
    p_digest.add_argument("--to", type=str,
                          default=os.environ.get("ME_TUNE_DIGEST_TO", ""),
                          help="Comma-separated recipient emails "
                               "(default: $ME_TUNE_DIGEST_TO)")
    p_digest.add_argument("--from", "--sender", dest="sender", type=str,
                          default=(os.environ.get("ME_TUNE_DIGEST_FROM")
                                   or "onboarding@resend.dev"),
                          help="Sender email (default: $ME_TUNE_DIGEST_FROM "
                               "or onboarding@resend.dev)")
    p_digest.add_argument("--window-days", type=int, default=7,
                          help="Window size in days (default: 7)")
    p_digest.add_argument("--dry-run", action="store_true",
                          help="Render HTML to stdout, do not send.")
    p_digest.set_defaults(func=_cmd_digest)

    return p


def main(argv: list[str] | None = None) -> int:
    parser = _build_parser()
    args = parser.parse_args(argv)
    return args.func(args)


if __name__ == "__main__":
    sys.exit(main())