Spaces:
Sleeping
Sleeping
| """argparse entry: me-hub feedback subcommands. | |
| Invoke as: python -m scripts.lib.chat_eval.cli <subcommand> [...] | |
| """ | |
| from __future__ import annotations | |
| import argparse | |
| import json | |
| import os | |
| import sys | |
| from datetime import datetime, timezone, timedelta | |
| from pathlib import Path | |
| from scripts.lib.chat_eval import analytics | |
| from scripts.lib.chat_eval.report import build_report | |
| from scripts.lib.chat_eval.state_io import fetch_state, push_state | |
| from scripts.lib.runtime_config import DERIVED_STUDENT_CHAT_DIR, TURN_LOG_PATH | |
| _DEFAULT_LOG = TURN_LOG_PATH | |
| def _print_table(headers: list[str], rows: list[list[str]]) -> None: | |
| if not rows: | |
| print("(no rows)") | |
| return | |
| print("| " + " | ".join(headers) + " |") | |
| print("|" + "|".join("---" for _ in headers) + "|") | |
| for row in rows: | |
| print("| " + " | ".join(str(c) for c in row) + " |") | |
| def _cmd_report(args) -> int: | |
| md = build_report(args.log_path, since=args.since) | |
| if args.out and args.out != "-": | |
| Path(args.out).write_text(md, encoding="utf-8") | |
| print(f"Report written to {args.out}") | |
| else: | |
| print(md) | |
| return 0 | |
| def _cmd_coverage(args) -> int: | |
| result = analytics.coverage(args.log_path, since=args.since) | |
| if args.json: | |
| print(json.dumps(result, indent=2)) | |
| return 0 | |
| rows = sorted(result.items(), key=lambda kv: -kv[1]) | |
| _print_table(["term", "count"], [[k, v] for k, v in rows]) | |
| return 0 | |
| def _cmd_misconceptions(args) -> int: | |
| result = analytics.misconception_frequency(args.log_path, since=args.since) | |
| if args.json: | |
| print(json.dumps(result, indent=2)) | |
| return 0 | |
| rows = sorted(result.items(), key=lambda kv: -kv[1]["count"]) | |
| _print_table( | |
| ["id", "count", "handled"], | |
| [[k, v["count"], v["handled"]] for k, v in rows], | |
| ) | |
| return 0 | |
| def _cmd_doc_utilization(args) -> int: | |
| result = analytics.doc_utilization(args.log_path, since=args.since) | |
| if args.json: | |
| print(json.dumps(result, indent=2)) | |
| return 0 | |
| rows = sorted(result.items(), | |
| key=lambda kv: (kv[1]["util_pct"], -kv[1]["retrieved_count"])) | |
| _print_table( | |
| ["doc_id", "retrieved", "used", "util %"], | |
| [[k, v["retrieved_count"], v["used_count"], v["util_pct"]] for k, v in rows], | |
| ) | |
| return 0 | |
| def _cmd_voice_failures(args) -> int: | |
| result = analytics.voice_failures(args.log_path, since=args.since) | |
| if args.json: | |
| print(json.dumps(result, indent=2)) | |
| return 0 | |
| if not result: | |
| print("(no voice failures)") | |
| return 0 | |
| for v in result: | |
| print(f"### {v['turn_id']} (voice {v['voice_score']})") | |
| print(f"- Query: {v['user_message']}") | |
| print(f"- Response: {v['response'][:200]}") | |
| print(f"- Diagnosis: {v['diagnosis']}") | |
| print() | |
| return 0 | |
| def _cmd_timing(args) -> int: | |
| result = analytics.timing_density(args.log_path, since=args.since) | |
| if args.json: | |
| print(json.dumps(result, indent=2)) | |
| return 0 | |
| print("| weekday | total |") | |
| print("|---|---|") | |
| for wd in ["Mon", "Tue", "Wed", "Thu", "Fri", "Sat", "Sun"]: | |
| print(f"| {wd} | {result.get(wd, {}).get('total', 0)} |") | |
| return 0 | |
| def _cmd_judge(args) -> int: | |
| """Run the judge over unjudged turns. Real implementation requires an | |
| Anthropic client and the catalog; assemble both and dispatch.""" | |
| from scripts.lib.catalog_loader import load_catalog | |
| from scripts.lib.chat_eval.judge import judge_turn, JudgeError, JudgeTransientError | |
| from scripts.lib.chat_eval.turn_log import ( | |
| is_student_mode_turn, iter_turns, update_judge, | |
| ) | |
| catalog = load_catalog(DERIVED_STUDENT_CHAT_DIR) | |
| turns = list(iter_turns(args.log_path, since=args.since)) | |
| unjudged = [ | |
| t for t in turns | |
| if t.get("judge") is None and is_student_mode_turn(t) | |
| ] | |
| if args.limit: | |
| unjudged = unjudged[: args.limit] | |
| if not unjudged: | |
| print("(no unjudged turns)") | |
| return 0 | |
| print(f"Judging {len(unjudged)} turns...") | |
| terminal = 0 | |
| transient = 0 | |
| for i, turn in enumerate(unjudged, 1): | |
| print(f"[{i}/{len(unjudged)}] {turn['turn_id']}") | |
| try: | |
| judge_dict = judge_turn(turn, catalog) | |
| update_judge(turn["turn_id"], judge_dict, args.log_path) | |
| except JudgeTransientError as e: | |
| # The turn stays unjudged; tomorrow's run will retry it. | |
| print(f" SKIPPED (transient): {e}") | |
| transient += 1 | |
| except JudgeError as e: | |
| print(f" FAILED: {e}") | |
| terminal += 1 | |
| judged = len(unjudged) - terminal - transient | |
| print( | |
| f"Done. {judged} judged, {transient} skipped (transient: will retry " | |
| f"next run), {terminal} failed (terminal)." | |
| ) | |
| return 0 if terminal == 0 else 1 | |
| def _cmd_notes(args) -> int: | |
| """Run the notes summarizer for each faculty login.""" | |
| from pathlib import Path as _P | |
| from scripts.lib.chat_eval import notes_io, notes_summarizer, tuning | |
| from scripts.lib.chat.prompt_builder import FACULTY_LOGINS | |
| missing = _missing_hf_env() | |
| if missing: | |
| print(f"ERROR: missing required env var(s): {', '.join(missing)}", | |
| file=sys.stderr) | |
| return 2 | |
| repo_id = os.environ["ME_CHAT_HISTORY_DATASET_REPO_ID"] | |
| token = os.environ["HF_TOKEN"] | |
| derived_dir = args.derived_dir | |
| turns_path = derived_dir / "turns.jsonl" | |
| archive_dir = derived_dir / "notes" / "archive" | |
| archive_dir.mkdir(parents=True, exist_ok=True) | |
| prior_rows = notes_io.fetch_summarizer_log(repo_id=repo_id, token=token) | |
| now_ts = tuning.now_utc_iso() | |
| llm = notes_summarizer.make_anthropic_client() | |
| overall_rows = list(prior_rows) | |
| had_terminal = False | |
| for login in sorted(FACULTY_LOGINS): | |
| print(f"[{login}] running summarizer...") | |
| current_body = notes_io.fetch_user_notes( | |
| login=login, repo_id=repo_id, token=token, | |
| ) | |
| try: | |
| new_body, audit = notes_summarizer.run_for_user( | |
| login, | |
| turns_path=turns_path, | |
| current_notes_body=current_body, | |
| prior_log_rows=overall_rows, | |
| llm=llm, | |
| now_ts=now_ts, | |
| ) | |
| except notes_summarizer.SummarizerTerminalError as e: | |
| print(f" TERMINAL: {e}", file=sys.stderr) | |
| had_terminal = True | |
| overall_rows.append({ | |
| "ts": now_ts, "login": login, "outcome": "terminal_failure", | |
| "since_ts": None, "last_processed_ts": None, | |
| "turns_processed": 0, | |
| "notes_chars_before": len(current_body), | |
| "notes_chars_after": None, "archived_to": None, | |
| "haiku_model": getattr(llm, "model_name", None), | |
| "elapsed_s": 0.0, "failure_reason": str(e), | |
| }) | |
| continue | |
| if new_body is None: | |
| print(f" outcome={audit['outcome']} (no notes write)") | |
| overall_rows.append(audit) | |
| continue | |
| archive_local = notes_io.next_archive_path_local( | |
| archive_dir=archive_dir, login=login, ts=now_ts, | |
| ) | |
| archive_local.write_text(current_body, encoding="utf-8") | |
| archive_repo_path = notes_io.archive_path_for(login=login, ts=now_ts) | |
| if archive_local.name != _P(archive_repo_path).name: | |
| # Collision case: archive_path_for ignores suffixes; rebuild | |
| # the repo path to match the local -NN suffix. | |
| archive_repo_path = f"notes/archive/{archive_local.name}" | |
| audit["archived_to"] = archive_repo_path | |
| overall_rows.append(audit) | |
| notes_io.push_notes_update( | |
| repo_id=repo_id, token=token, login=login, | |
| new_notes_body=new_body, | |
| archive_repo_path=archive_repo_path, | |
| archive_body=current_body, | |
| summarizer_log_rows=overall_rows, | |
| commit_message=f"notes: {login} {now_ts[:10]}", | |
| ) | |
| print(f" outcome=updated (chars {audit['notes_chars_before']} → {audit['notes_chars_after']})") | |
| # If no successful update happened, still flush the log so no-op / | |
| # transient rows reach the dataset. | |
| if overall_rows != prior_rows: | |
| notes_io.push_summarizer_log_only( | |
| repo_id=repo_id, token=token, | |
| summarizer_log_rows=overall_rows, | |
| commit_message=f"notes: {now_ts[:10]} audit", | |
| ) | |
| return 1 if had_terminal else 0 | |
| def _cmd_tune(args) -> int: | |
| from scripts.lib.chat_eval.tune_runner import run | |
| return run(args) | |
| def _missing_hf_env() -> list[str]: | |
| """Return the list of HF Dataset env vars that are unset (empty list if all present).""" | |
| missing = [] | |
| if not os.environ.get("ME_CHAT_HISTORY_DATASET_REPO_ID"): | |
| missing.append("ME_CHAT_HISTORY_DATASET_REPO_ID") | |
| if not os.environ.get("HF_TOKEN"): | |
| missing.append("HF_TOKEN") | |
| return missing | |
| def _cmd_fetch_state(args) -> int: | |
| missing = _missing_hf_env() | |
| if missing: | |
| print(f"ERROR: missing required env var(s): {', '.join(missing)}", | |
| file=sys.stderr) | |
| return 2 | |
| fetch_state(repo_id=os.environ["ME_CHAT_HISTORY_DATASET_REPO_ID"], | |
| token=os.environ["HF_TOKEN"], | |
| derived_dir=args.derived_dir) | |
| return 0 | |
| def _cmd_push_state(args) -> int: | |
| missing = _missing_hf_env() | |
| if missing: | |
| print(f"ERROR: missing required env var(s): {', '.join(missing)}", | |
| file=sys.stderr) | |
| return 2 | |
| push_state(repo_id=os.environ["ME_CHAT_HISTORY_DATASET_REPO_ID"], | |
| token=os.environ["HF_TOKEN"], | |
| derived_dir=args.derived_dir, | |
| commit_message=args.commit_message) | |
| return 0 | |
| def _cmd_digest(args) -> int: | |
| """Build and optionally send the weekly HTML digest.""" | |
| from scripts.lib.chat_eval import tuning | |
| from scripts.lib.chat_eval.digest import ( | |
| faculty_digest, | |
| render_html, | |
| send_via_resend, | |
| ) | |
| now_ts = tuning.now_utc_iso() | |
| now_dt = datetime.strptime(now_ts, "%Y-%m-%dT%H:%M:%SZ").replace(tzinfo=timezone.utc) | |
| since_dt = now_dt - timedelta(days=args.window_days) | |
| since_iso = since_dt.strftime("%Y-%m-%dT%H:%M:%SZ") | |
| today = now_dt.date().isoformat() | |
| body_md = faculty_digest( | |
| args.turns_log, | |
| args.tuning_log, | |
| since=since_iso, | |
| today=today, | |
| window_days=args.window_days, | |
| ) | |
| html = render_html(body_md) | |
| if args.dry_run: | |
| sys.stdout.write(html) | |
| if not html.endswith("\n"): | |
| sys.stdout.write("\n") | |
| return 0 | |
| api_key = os.environ.get("RESEND_API_KEY") | |
| if not api_key: | |
| print("ERROR: RESEND_API_KEY not set", file=sys.stderr) | |
| return 2 | |
| recipients = [r.strip() for r in (args.to or "").split(",") if r.strip()] | |
| if not recipients: | |
| print("ERROR: --to must contain at least one email address", | |
| file=sys.stderr) | |
| return 2 | |
| sender = args.sender or "onboarding@resend.dev" | |
| subject = f"MGRECON780 chat tutor — weekly briefing {today}" | |
| send_via_resend( | |
| api_key=api_key, | |
| sender=sender, | |
| recipients=recipients, | |
| subject=subject, | |
| html_body=html, | |
| ) | |
| print(f"Sent digest to {len(recipients)} recipient(s).") | |
| return 0 | |
| def _build_parser() -> argparse.ArgumentParser: | |
| p = argparse.ArgumentParser(prog="python -m scripts.lib.chat_eval.cli") | |
| sub = p.add_subparsers(dest="command", required=True) | |
| def _common(parser): | |
| parser.add_argument("--log-path", type=Path, default=_DEFAULT_LOG, | |
| help="Path to turns.jsonl (default: derived/student-chat/turns.jsonl)") | |
| parser.add_argument("--since", type=str, default=None, | |
| help="ISO date filter (YYYY-MM-DD)") | |
| parser.add_argument("--json", action="store_true", | |
| help="Output JSON instead of markdown table") | |
| p_report = sub.add_parser("report", help="Generate unified weekly report") | |
| _common(p_report) | |
| p_report.add_argument("--out", type=str, default="-", | |
| help="Output path (default: stdout)") | |
| p_report.set_defaults(func=_cmd_report) | |
| p_coverage = sub.add_parser("coverage", help="Concept coverage") | |
| _common(p_coverage) | |
| p_coverage.set_defaults(func=_cmd_coverage) | |
| p_mc = sub.add_parser("misconceptions", help="Misconception frequency") | |
| _common(p_mc) | |
| p_mc.set_defaults(func=_cmd_misconceptions) | |
| p_du = sub.add_parser("doc-utilization", help="Per-doc utilization") | |
| _common(p_du) | |
| p_du.set_defaults(func=_cmd_doc_utilization) | |
| p_vf = sub.add_parser("voice-failures", help="Turns with low voice scores") | |
| _common(p_vf) | |
| p_vf.set_defaults(func=_cmd_voice_failures) | |
| p_t = sub.add_parser("timing", help="Weekday/hour density") | |
| _common(p_t) | |
| p_t.set_defaults(func=_cmd_timing) | |
| p_judge = sub.add_parser("judge", help="Run judge over unjudged turns") | |
| _common(p_judge) | |
| p_judge.add_argument("--limit", type=int, default=None, | |
| help="Max turns to judge in this run") | |
| p_judge.set_defaults(func=_cmd_judge) | |
| p_tune = sub.add_parser("tune", | |
| help="Propose one tuning step") | |
| p_tune.add_argument("--log-path", type=Path, default=_DEFAULT_LOG, | |
| help="Path to turns.jsonl") | |
| p_tune.add_argument("--tuning-state-path", type=Path, | |
| default=_DEFAULT_LOG.parent / "retrieval_tuning.json") | |
| p_tune.add_argument("--tuning-log-path", type=Path, | |
| default=_DEFAULT_LOG.parent / "tuning_log.jsonl") | |
| p_tune.add_argument("--unfreeze", action="store_true", | |
| help="Clear frozen flag; do not propose a step.") | |
| p_tune.add_argument("--dry-run", action="store_true", | |
| help="Compute the step but do not write state.") | |
| p_tune.set_defaults(func=_cmd_tune) | |
| p_fs = sub.add_parser("fetch-state", | |
| help="Download tuning state files from HF Dataset") | |
| p_fs.add_argument("--derived-dir", type=Path, | |
| default=_DEFAULT_LOG.parent) | |
| p_fs.set_defaults(func=_cmd_fetch_state) | |
| p_ps = sub.add_parser("push-state", | |
| help="Upload tuning state files to HF Dataset") | |
| p_ps.add_argument("--derived-dir", type=Path, | |
| default=_DEFAULT_LOG.parent) | |
| p_ps.add_argument("--commit-message", type=str, | |
| default="tuner: update state") | |
| p_ps.set_defaults(func=_cmd_push_state) | |
| p_notes = sub.add_parser( | |
| "notes", | |
| help="Run the per-faculty notes summarizer", | |
| ) | |
| p_notes.add_argument("--derived-dir", type=Path, | |
| default=_DEFAULT_LOG.parent, | |
| help="Working dir for turns.jsonl + notes/ (default: derived/student-chat)") | |
| p_notes.set_defaults(func=_cmd_notes) | |
| p_digest = sub.add_parser( | |
| "digest", | |
| help="Build the weekly HTML digest and send via Resend", | |
| ) | |
| p_digest.add_argument("--turns-log", type=Path, default=_DEFAULT_LOG, | |
| help="Path to turns.jsonl") | |
| p_digest.add_argument("--tuning-log", type=Path, | |
| default=_DEFAULT_LOG.parent / "tuning_log.jsonl", | |
| help="Path to tuning_log.jsonl") | |
| p_digest.add_argument("--to", type=str, | |
| default=os.environ.get("ME_TUNE_DIGEST_TO", ""), | |
| help="Comma-separated recipient emails " | |
| "(default: $ME_TUNE_DIGEST_TO)") | |
| p_digest.add_argument("--from", "--sender", dest="sender", type=str, | |
| default=(os.environ.get("ME_TUNE_DIGEST_FROM") | |
| or "onboarding@resend.dev"), | |
| help="Sender email (default: $ME_TUNE_DIGEST_FROM " | |
| "or onboarding@resend.dev)") | |
| p_digest.add_argument("--window-days", type=int, default=7, | |
| help="Window size in days (default: 7)") | |
| p_digest.add_argument("--dry-run", action="store_true", | |
| help="Render HTML to stdout, do not send.") | |
| p_digest.set_defaults(func=_cmd_digest) | |
| return p | |
| def main(argv: list[str] | None = None) -> int: | |
| parser = _build_parser() | |
| args = parser.parse_args(argv) | |
| return args.func(args) | |
| if __name__ == "__main__": | |
| sys.exit(main()) | |