"""argparse entry: me-hub feedback subcommands. Invoke as: python -m scripts.lib.chat_eval.cli [...] """ from __future__ import annotations import argparse import json import os import sys from datetime import datetime, timezone, timedelta from pathlib import Path from scripts.lib.chat_eval import analytics from scripts.lib.chat_eval.report import build_report from scripts.lib.chat_eval.state_io import fetch_state, push_state from scripts.lib.runtime_config import DERIVED_STUDENT_CHAT_DIR, TURN_LOG_PATH _DEFAULT_LOG = TURN_LOG_PATH def _print_table(headers: list[str], rows: list[list[str]]) -> None: if not rows: print("(no rows)") return print("| " + " | ".join(headers) + " |") print("|" + "|".join("---" for _ in headers) + "|") for row in rows: print("| " + " | ".join(str(c) for c in row) + " |") def _cmd_report(args) -> int: md = build_report(args.log_path, since=args.since) if args.out and args.out != "-": Path(args.out).write_text(md, encoding="utf-8") print(f"Report written to {args.out}") else: print(md) return 0 def _cmd_coverage(args) -> int: result = analytics.coverage(args.log_path, since=args.since) if args.json: print(json.dumps(result, indent=2)) return 0 rows = sorted(result.items(), key=lambda kv: -kv[1]) _print_table(["term", "count"], [[k, v] for k, v in rows]) return 0 def _cmd_misconceptions(args) -> int: result = analytics.misconception_frequency(args.log_path, since=args.since) if args.json: print(json.dumps(result, indent=2)) return 0 rows = sorted(result.items(), key=lambda kv: -kv[1]["count"]) _print_table( ["id", "count", "handled"], [[k, v["count"], v["handled"]] for k, v in rows], ) return 0 def _cmd_doc_utilization(args) -> int: result = analytics.doc_utilization(args.log_path, since=args.since) if args.json: print(json.dumps(result, indent=2)) return 0 rows = sorted(result.items(), key=lambda kv: (kv[1]["util_pct"], -kv[1]["retrieved_count"])) _print_table( ["doc_id", "retrieved", "used", "util %"], [[k, v["retrieved_count"], v["used_count"], v["util_pct"]] for k, v in rows], ) return 0 def _cmd_voice_failures(args) -> int: result = analytics.voice_failures(args.log_path, since=args.since) if args.json: print(json.dumps(result, indent=2)) return 0 if not result: print("(no voice failures)") return 0 for v in result: print(f"### {v['turn_id']} (voice {v['voice_score']})") print(f"- Query: {v['user_message']}") print(f"- Response: {v['response'][:200]}") print(f"- Diagnosis: {v['diagnosis']}") print() return 0 def _cmd_timing(args) -> int: result = analytics.timing_density(args.log_path, since=args.since) if args.json: print(json.dumps(result, indent=2)) return 0 print("| weekday | total |") print("|---|---|") for wd in ["Mon", "Tue", "Wed", "Thu", "Fri", "Sat", "Sun"]: print(f"| {wd} | {result.get(wd, {}).get('total', 0)} |") return 0 def _cmd_judge(args) -> int: """Run the judge over unjudged turns. Real implementation requires an Anthropic client and the catalog; assemble both and dispatch.""" from scripts.lib.catalog_loader import load_catalog from scripts.lib.chat_eval.judge import judge_turn, JudgeError, JudgeTransientError from scripts.lib.chat_eval.turn_log import ( is_student_mode_turn, iter_turns, update_judge, ) catalog = load_catalog(DERIVED_STUDENT_CHAT_DIR) turns = list(iter_turns(args.log_path, since=args.since)) unjudged = [ t for t in turns if t.get("judge") is None and is_student_mode_turn(t) ] if args.limit: unjudged = unjudged[: args.limit] if not unjudged: print("(no unjudged turns)") return 0 print(f"Judging {len(unjudged)} turns...") terminal = 0 transient = 0 for i, turn in enumerate(unjudged, 1): print(f"[{i}/{len(unjudged)}] {turn['turn_id']}") try: judge_dict = judge_turn(turn, catalog) update_judge(turn["turn_id"], judge_dict, args.log_path) except JudgeTransientError as e: # The turn stays unjudged; tomorrow's run will retry it. print(f" SKIPPED (transient): {e}") transient += 1 except JudgeError as e: print(f" FAILED: {e}") terminal += 1 judged = len(unjudged) - terminal - transient print( f"Done. {judged} judged, {transient} skipped (transient: will retry " f"next run), {terminal} failed (terminal)." ) return 0 if terminal == 0 else 1 def _cmd_notes(args) -> int: """Run the notes summarizer for each faculty login.""" from pathlib import Path as _P from scripts.lib.chat_eval import notes_io, notes_summarizer, tuning from scripts.lib.chat.prompt_builder import FACULTY_LOGINS missing = _missing_hf_env() if missing: print(f"ERROR: missing required env var(s): {', '.join(missing)}", file=sys.stderr) return 2 repo_id = os.environ["ME_CHAT_HISTORY_DATASET_REPO_ID"] token = os.environ["HF_TOKEN"] derived_dir = args.derived_dir turns_path = derived_dir / "turns.jsonl" archive_dir = derived_dir / "notes" / "archive" archive_dir.mkdir(parents=True, exist_ok=True) prior_rows = notes_io.fetch_summarizer_log(repo_id=repo_id, token=token) now_ts = tuning.now_utc_iso() llm = notes_summarizer.make_anthropic_client() overall_rows = list(prior_rows) had_terminal = False for login in sorted(FACULTY_LOGINS): print(f"[{login}] running summarizer...") current_body = notes_io.fetch_user_notes( login=login, repo_id=repo_id, token=token, ) try: new_body, audit = notes_summarizer.run_for_user( login, turns_path=turns_path, current_notes_body=current_body, prior_log_rows=overall_rows, llm=llm, now_ts=now_ts, ) except notes_summarizer.SummarizerTerminalError as e: print(f" TERMINAL: {e}", file=sys.stderr) had_terminal = True overall_rows.append({ "ts": now_ts, "login": login, "outcome": "terminal_failure", "since_ts": None, "last_processed_ts": None, "turns_processed": 0, "notes_chars_before": len(current_body), "notes_chars_after": None, "archived_to": None, "haiku_model": getattr(llm, "model_name", None), "elapsed_s": 0.0, "failure_reason": str(e), }) continue if new_body is None: print(f" outcome={audit['outcome']} (no notes write)") overall_rows.append(audit) continue archive_local = notes_io.next_archive_path_local( archive_dir=archive_dir, login=login, ts=now_ts, ) archive_local.write_text(current_body, encoding="utf-8") archive_repo_path = notes_io.archive_path_for(login=login, ts=now_ts) if archive_local.name != _P(archive_repo_path).name: # Collision case: archive_path_for ignores suffixes; rebuild # the repo path to match the local -NN suffix. archive_repo_path = f"notes/archive/{archive_local.name}" audit["archived_to"] = archive_repo_path overall_rows.append(audit) notes_io.push_notes_update( repo_id=repo_id, token=token, login=login, new_notes_body=new_body, archive_repo_path=archive_repo_path, archive_body=current_body, summarizer_log_rows=overall_rows, commit_message=f"notes: {login} {now_ts[:10]}", ) print(f" outcome=updated (chars {audit['notes_chars_before']} → {audit['notes_chars_after']})") # If no successful update happened, still flush the log so no-op / # transient rows reach the dataset. if overall_rows != prior_rows: notes_io.push_summarizer_log_only( repo_id=repo_id, token=token, summarizer_log_rows=overall_rows, commit_message=f"notes: {now_ts[:10]} audit", ) return 1 if had_terminal else 0 def _cmd_tune(args) -> int: from scripts.lib.chat_eval.tune_runner import run return run(args) def _missing_hf_env() -> list[str]: """Return the list of HF Dataset env vars that are unset (empty list if all present).""" missing = [] if not os.environ.get("ME_CHAT_HISTORY_DATASET_REPO_ID"): missing.append("ME_CHAT_HISTORY_DATASET_REPO_ID") if not os.environ.get("HF_TOKEN"): missing.append("HF_TOKEN") return missing def _cmd_fetch_state(args) -> int: missing = _missing_hf_env() if missing: print(f"ERROR: missing required env var(s): {', '.join(missing)}", file=sys.stderr) return 2 fetch_state(repo_id=os.environ["ME_CHAT_HISTORY_DATASET_REPO_ID"], token=os.environ["HF_TOKEN"], derived_dir=args.derived_dir) return 0 def _cmd_push_state(args) -> int: missing = _missing_hf_env() if missing: print(f"ERROR: missing required env var(s): {', '.join(missing)}", file=sys.stderr) return 2 push_state(repo_id=os.environ["ME_CHAT_HISTORY_DATASET_REPO_ID"], token=os.environ["HF_TOKEN"], derived_dir=args.derived_dir, commit_message=args.commit_message) return 0 def _cmd_digest(args) -> int: """Build and optionally send the weekly HTML digest.""" from scripts.lib.chat_eval import tuning from scripts.lib.chat_eval.digest import ( faculty_digest, render_html, send_via_resend, ) now_ts = tuning.now_utc_iso() now_dt = datetime.strptime(now_ts, "%Y-%m-%dT%H:%M:%SZ").replace(tzinfo=timezone.utc) since_dt = now_dt - timedelta(days=args.window_days) since_iso = since_dt.strftime("%Y-%m-%dT%H:%M:%SZ") today = now_dt.date().isoformat() body_md = faculty_digest( args.turns_log, args.tuning_log, since=since_iso, today=today, window_days=args.window_days, ) html = render_html(body_md) if args.dry_run: sys.stdout.write(html) if not html.endswith("\n"): sys.stdout.write("\n") return 0 api_key = os.environ.get("RESEND_API_KEY") if not api_key: print("ERROR: RESEND_API_KEY not set", file=sys.stderr) return 2 recipients = [r.strip() for r in (args.to or "").split(",") if r.strip()] if not recipients: print("ERROR: --to must contain at least one email address", file=sys.stderr) return 2 sender = args.sender or "onboarding@resend.dev" subject = f"MGRECON780 chat tutor — weekly briefing {today}" send_via_resend( api_key=api_key, sender=sender, recipients=recipients, subject=subject, html_body=html, ) print(f"Sent digest to {len(recipients)} recipient(s).") return 0 def _build_parser() -> argparse.ArgumentParser: p = argparse.ArgumentParser(prog="python -m scripts.lib.chat_eval.cli") sub = p.add_subparsers(dest="command", required=True) def _common(parser): parser.add_argument("--log-path", type=Path, default=_DEFAULT_LOG, help="Path to turns.jsonl (default: derived/student-chat/turns.jsonl)") parser.add_argument("--since", type=str, default=None, help="ISO date filter (YYYY-MM-DD)") parser.add_argument("--json", action="store_true", help="Output JSON instead of markdown table") p_report = sub.add_parser("report", help="Generate unified weekly report") _common(p_report) p_report.add_argument("--out", type=str, default="-", help="Output path (default: stdout)") p_report.set_defaults(func=_cmd_report) p_coverage = sub.add_parser("coverage", help="Concept coverage") _common(p_coverage) p_coverage.set_defaults(func=_cmd_coverage) p_mc = sub.add_parser("misconceptions", help="Misconception frequency") _common(p_mc) p_mc.set_defaults(func=_cmd_misconceptions) p_du = sub.add_parser("doc-utilization", help="Per-doc utilization") _common(p_du) p_du.set_defaults(func=_cmd_doc_utilization) p_vf = sub.add_parser("voice-failures", help="Turns with low voice scores") _common(p_vf) p_vf.set_defaults(func=_cmd_voice_failures) p_t = sub.add_parser("timing", help="Weekday/hour density") _common(p_t) p_t.set_defaults(func=_cmd_timing) p_judge = sub.add_parser("judge", help="Run judge over unjudged turns") _common(p_judge) p_judge.add_argument("--limit", type=int, default=None, help="Max turns to judge in this run") p_judge.set_defaults(func=_cmd_judge) p_tune = sub.add_parser("tune", help="Propose one tuning step") p_tune.add_argument("--log-path", type=Path, default=_DEFAULT_LOG, help="Path to turns.jsonl") p_tune.add_argument("--tuning-state-path", type=Path, default=_DEFAULT_LOG.parent / "retrieval_tuning.json") p_tune.add_argument("--tuning-log-path", type=Path, default=_DEFAULT_LOG.parent / "tuning_log.jsonl") p_tune.add_argument("--unfreeze", action="store_true", help="Clear frozen flag; do not propose a step.") p_tune.add_argument("--dry-run", action="store_true", help="Compute the step but do not write state.") p_tune.set_defaults(func=_cmd_tune) p_fs = sub.add_parser("fetch-state", help="Download tuning state files from HF Dataset") p_fs.add_argument("--derived-dir", type=Path, default=_DEFAULT_LOG.parent) p_fs.set_defaults(func=_cmd_fetch_state) p_ps = sub.add_parser("push-state", help="Upload tuning state files to HF Dataset") p_ps.add_argument("--derived-dir", type=Path, default=_DEFAULT_LOG.parent) p_ps.add_argument("--commit-message", type=str, default="tuner: update state") p_ps.set_defaults(func=_cmd_push_state) p_notes = sub.add_parser( "notes", help="Run the per-faculty notes summarizer", ) p_notes.add_argument("--derived-dir", type=Path, default=_DEFAULT_LOG.parent, help="Working dir for turns.jsonl + notes/ (default: derived/student-chat)") p_notes.set_defaults(func=_cmd_notes) p_digest = sub.add_parser( "digest", help="Build the weekly HTML digest and send via Resend", ) p_digest.add_argument("--turns-log", type=Path, default=_DEFAULT_LOG, help="Path to turns.jsonl") p_digest.add_argument("--tuning-log", type=Path, default=_DEFAULT_LOG.parent / "tuning_log.jsonl", help="Path to tuning_log.jsonl") p_digest.add_argument("--to", type=str, default=os.environ.get("ME_TUNE_DIGEST_TO", ""), help="Comma-separated recipient emails " "(default: $ME_TUNE_DIGEST_TO)") p_digest.add_argument("--from", "--sender", dest="sender", type=str, default=(os.environ.get("ME_TUNE_DIGEST_FROM") or "onboarding@resend.dev"), help="Sender email (default: $ME_TUNE_DIGEST_FROM " "or onboarding@resend.dev)") p_digest.add_argument("--window-days", type=int, default=7, help="Window size in days (default: 7)") p_digest.add_argument("--dry-run", action="store_true", help="Render HTML to stdout, do not send.") p_digest.set_defaults(func=_cmd_digest) return p def main(argv: list[str] | None = None) -> int: parser = _build_parser() args = parser.parse_args(argv) return args.func(args) if __name__ == "__main__": sys.exit(main())