"""Inspect and grade what testers generated. python -m api.review summary counts per kind: auto-valid, tester-rated, reviewed python -m api.review list [--kind simulate|code] [--tester NAME] [--unreviewed] [-n 50] python -m api.review show ID [--attempts] request, retrieved sources, output, validation python -m api.review files ID DIR write the output to DIR (scene.xml or the package files) python -m api.review mark ID correct|partial|incorrect [--note TEXT] python -m api.review export out.jsonl everything, one JSON object per line python -m api.review runs [--tester NAME] [-n 50] every /simulate/run call: who, when, status, GPU time python -m api.review usage runs per tester: total, last 24 h, busiest 10 minutes """ from __future__ import annotations import argparse import json import sys from pathlib import Path from .config import get_settings from .logstore import LogStore def _short(text: str, n: int = 60) -> str: text = " ".join(str(text).split()) return text if len(text) <= n else text[: n - 1] + "…" def _request_text(row: dict) -> str: req = row["request"] or {} return req.get("description") or req.get("request") or "" def _auto(row: dict) -> str: if row.get("error"): return "ERROR" return {True: "valid", False: "invalid", None: "-"}[row.get("valid")] def cmd_list(logs: LogStore, args) -> int: rows = logs.list(args.kind, args.tester, args.unreviewed, args.n) header = ("id", 12), ("created (UTC)", 19), ("tester", 10), ("kind", 8), ("auto", 7), ("tester", 7), ("review", 9) print(" ".join(f"{h:<{w}}" for h, w in header) + " request") for r in rows: print( f"{r['id']:<12} {r['created_at'][:19]:<19} {_short(r['tester'], 10):<10} {r['kind']:<8} " f"{_auto(r):<7} {r['tester_rating'] or '-':<7} {r['review_verdict'] or '-':<9} {_short(_request_text(r))}" ) if not rows: print("(no generations)") return 0 def cmd_show(logs: LogStore, args) -> int: r = logs.get(args.id) if not r: print(f"no generation {args.id}", file=sys.stderr) return 1 print(f"id: {r['id']} kind: {r['kind']} tester: {r['tester']} at: {r['created_at']}") print(f"model: {r['model']} duration: {r['duration_ms']} ms attempts: {len(r['attempts'] or [])}") print(f"request: {json.dumps(r['request'])}") print(f"auto: {_auto(r)} {json.dumps(r['validation'])}") if r.get("error"): print(f"error: {r['error']}") print(f"tester: {r['tester_rating'] or '-'} {r['tester_note'] or ''}") print(f"review: {r['review_verdict'] or '-'} {r['review_note'] or ''}") print("sources:") for s in r["sources"] or []: print(f" - {s}") if args.attempts: for i, a in enumerate(r["attempts"] or [], 1): print(f"\n----- attempt {i}: {json.dumps(a.get('validation'))}") print(a.get("raw_output", "")) print("\n----- output") if r["kind"] == "code" and r["output"]: for path, content in json.loads(r["output"]).items(): print(f"\n=== {path}\n{content}") else: print(r["output"] or "(none)") return 0 def cmd_files(logs: LogStore, args) -> int: r = logs.get(args.id) if not r or not r["output"]: print(f"no output stored for {args.id}", file=sys.stderr) return 1 out = Path(args.dir) out.mkdir(parents=True, exist_ok=True) files = json.loads(r["output"]) if r["kind"] == "code" else {"scene_rosdiff.xml": r["output"]} for path, content in files.items(): dest = (out / path).resolve() if out.resolve() not in dest.parents: print(f"skipping unsafe path {path}", file=sys.stderr) continue dest.parent.mkdir(parents=True, exist_ok=True) dest.write_text(content) print(f"wrote {dest}") return 0 def cmd_mark(logs: LogStore, args) -> int: if not logs.set_review(args.id, args.verdict, args.note): print(f"no generation {args.id}", file=sys.stderr) return 1 print(f"{args.id}: {args.verdict}") return 0 def cmd_summary(logs: LogStore, args) -> int: rows = logs.summary() if not rows: print("(no generations)") for r in rows: print( f"{r['kind']:<9} total {r['total']:>4} | auto-valid {r['auto_valid'] or 0:>4} | " f"tester says works {r['tester_works'] or 0}/{r['tester_rated'] or 0} | " f"reviewed correct {r['reviewed_correct'] or 0}/{r['reviewed'] or 0}" ) return 0 def cmd_export(logs: LogStore, args) -> int: with open(args.path, "w") as f: for r in logs.all(): f.write(json.dumps(r) + "\n") print(f"wrote {args.path}") return 0 def cmd_runs(logs: LogStore, args) -> int: rows = logs.list_runs(args.tester, args.n) header = ("job", 16), ("kind", 9), ("created (UTC)", 19), ("tester", 12), ("status", 11), ("sim s", 5) header += (("run s", 6), ("cost $", 6)) print(" ".join(f"{h:<{w}}" for h, w in header) + " source") for r in rows: gpu = f"{r['execution_ms'] / 1000:.1f}" if r.get("execution_ms") else "-" cost = f"{r['cost_usd']:.3f}" if r.get("cost_usd") is not None else "-" if r.get("parent_id"): source = f"run {r['parent_id']}" else: source = f"generation {r['generation_id']}" if r.get("generation_id") else "raw input" print( f"{r['id']:<16} {(r.get('kind') or 'sim'):<9} {r['created_at'][:19]:<19} {_short(r['tester'], 12):<12} " f"{r['status']:<11} {r['duration_s'] or 0:<5g} {gpu:<6} {cost:<6} {source}" ) if not rows: print("(no runs)") return 0 def cmd_usage(logs: LogStore, args) -> int: from .jobs import month_spend settings = get_settings() rows = logs.usage() spent, limit = month_spend(settings, logs), settings.monthly_credit_usd print(f"this month (estimated): ${spent:.2f}" + (f" of ${limit:.2f} limit" if limit else " (no limit)")) if not rows: print("(no jobs)") for u in rows: cost = f" ~${u['cost_usd']:.2f}" flag = " <-- burst" if u["max_in_10min"] >= settings.run_burst_warn else "" print( f"{u['tester']:<16} runs {u['runs']:>4} last 24h {u['last_24h']:>4} busiest 10 min " f"{u['max_in_10min']:>3} GPU {u['gpu_seconds']:>7.1f} s{cost} last {u['last'][:19]}{flag}" ) return 0 def main(argv: list[str] | None = None) -> int: parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) parser.add_argument("--db", type=Path, help="default: $ROSDIFF_LOG_DB or data/generations.sqlite3") sub = parser.add_subparsers(dest="cmd", required=True) sub.add_parser("summary") p = sub.add_parser("list") p.add_argument("--kind", choices=["simulate", "code"]) p.add_argument("--tester") p.add_argument("--unreviewed", action="store_true") p.add_argument("-n", type=int, default=50) p = sub.add_parser("show") p.add_argument("id") p.add_argument("--attempts", action="store_true", help="also print every raw model attempt") p = sub.add_parser("files") p.add_argument("id") p.add_argument("dir") p = sub.add_parser("mark") p.add_argument("id") p.add_argument("verdict", choices=["correct", "partial", "incorrect"]) p.add_argument("--note", default="") p = sub.add_parser("export") p.add_argument("path") p = sub.add_parser("runs") p.add_argument("--tester") p.add_argument("-n", type=int, default=50) sub.add_parser("usage") args = parser.parse_args(argv) logs = LogStore(args.db or get_settings().log_db) handlers = { "list": cmd_list, "show": cmd_show, "files": cmd_files, "mark": cmd_mark, "summary": cmd_summary, "export": cmd_export, "runs": cmd_runs, "usage": cmd_usage, } return handlers[args.cmd](logs, args) if __name__ == "__main__": sys.exit(main())