Spaces:
Paused
Paused
| """Inspect and grade what testers generated. | |
| python -m api.review summary counts per kind: auto-valid, tester-rated, reviewed | |
| python -m api.review list [--kind simulate|code] [--tester NAME] [--unreviewed] [-n 50] | |
| python -m api.review show ID [--attempts] request, retrieved sources, output, validation | |
| python -m api.review files ID DIR write the output to DIR (scene.xml or the package files) | |
| python -m api.review mark ID correct|partial|incorrect [--note TEXT] | |
| python -m api.review export out.jsonl everything, one JSON object per line | |
| python -m api.review runs [--tester NAME] [-n 50] every /simulate/run call: who, when, status, GPU time | |
| python -m api.review usage runs per tester: total, last 24 h, busiest 10 minutes | |
| """ | |
| from __future__ import annotations | |
| import argparse | |
| import json | |
| import sys | |
| from pathlib import Path | |
| from .config import get_settings | |
| from .logstore import LogStore | |
| def _short(text: str, n: int = 60) -> str: | |
| text = " ".join(str(text).split()) | |
| return text if len(text) <= n else text[: n - 1] + "…" | |
| def _request_text(row: dict) -> str: | |
| req = row["request"] or {} | |
| return req.get("description") or req.get("request") or "" | |
| def _auto(row: dict) -> str: | |
| if row.get("error"): | |
| return "ERROR" | |
| return {True: "valid", False: "invalid", None: "-"}[row.get("valid")] | |
| def cmd_list(logs: LogStore, args) -> int: | |
| rows = logs.list(args.kind, args.tester, args.unreviewed, args.n) | |
| header = ("id", 12), ("created (UTC)", 19), ("tester", 10), ("kind", 8), ("auto", 7), ("tester", 7), ("review", 9) | |
| print(" ".join(f"{h:<{w}}" for h, w in header) + " request") | |
| for r in rows: | |
| print( | |
| f"{r['id']:<12} {r['created_at'][:19]:<19} {_short(r['tester'], 10):<10} {r['kind']:<8} " | |
| f"{_auto(r):<7} {r['tester_rating'] or '-':<7} {r['review_verdict'] or '-':<9} {_short(_request_text(r))}" | |
| ) | |
| if not rows: | |
| print("(no generations)") | |
| return 0 | |
| def cmd_show(logs: LogStore, args) -> int: | |
| r = logs.get(args.id) | |
| if not r: | |
| print(f"no generation {args.id}", file=sys.stderr) | |
| return 1 | |
| print(f"id: {r['id']} kind: {r['kind']} tester: {r['tester']} at: {r['created_at']}") | |
| print(f"model: {r['model']} duration: {r['duration_ms']} ms attempts: {len(r['attempts'] or [])}") | |
| print(f"request: {json.dumps(r['request'])}") | |
| print(f"auto: {_auto(r)} {json.dumps(r['validation'])}") | |
| if r.get("error"): | |
| print(f"error: {r['error']}") | |
| print(f"tester: {r['tester_rating'] or '-'} {r['tester_note'] or ''}") | |
| print(f"review: {r['review_verdict'] or '-'} {r['review_note'] or ''}") | |
| print("sources:") | |
| for s in r["sources"] or []: | |
| print(f" - {s}") | |
| if args.attempts: | |
| for i, a in enumerate(r["attempts"] or [], 1): | |
| print(f"\n----- attempt {i}: {json.dumps(a.get('validation'))}") | |
| print(a.get("raw_output", "")) | |
| print("\n----- output") | |
| if r["kind"] == "code" and r["output"]: | |
| for path, content in json.loads(r["output"]).items(): | |
| print(f"\n=== {path}\n{content}") | |
| else: | |
| print(r["output"] or "(none)") | |
| return 0 | |
| def cmd_files(logs: LogStore, args) -> int: | |
| r = logs.get(args.id) | |
| if not r or not r["output"]: | |
| print(f"no output stored for {args.id}", file=sys.stderr) | |
| return 1 | |
| out = Path(args.dir) | |
| out.mkdir(parents=True, exist_ok=True) | |
| files = json.loads(r["output"]) if r["kind"] == "code" else {"scene_rosdiff.xml": r["output"]} | |
| for path, content in files.items(): | |
| dest = (out / path).resolve() | |
| if out.resolve() not in dest.parents: | |
| print(f"skipping unsafe path {path}", file=sys.stderr) | |
| continue | |
| dest.parent.mkdir(parents=True, exist_ok=True) | |
| dest.write_text(content) | |
| print(f"wrote {dest}") | |
| return 0 | |
| def cmd_mark(logs: LogStore, args) -> int: | |
| if not logs.set_review(args.id, args.verdict, args.note): | |
| print(f"no generation {args.id}", file=sys.stderr) | |
| return 1 | |
| print(f"{args.id}: {args.verdict}") | |
| return 0 | |
| def cmd_summary(logs: LogStore, args) -> int: | |
| rows = logs.summary() | |
| if not rows: | |
| print("(no generations)") | |
| for r in rows: | |
| print( | |
| f"{r['kind']:<9} total {r['total']:>4} | auto-valid {r['auto_valid'] or 0:>4} | " | |
| f"tester says works {r['tester_works'] or 0}/{r['tester_rated'] or 0} | " | |
| f"reviewed correct {r['reviewed_correct'] or 0}/{r['reviewed'] or 0}" | |
| ) | |
| return 0 | |
| def cmd_export(logs: LogStore, args) -> int: | |
| with open(args.path, "w") as f: | |
| for r in logs.all(): | |
| f.write(json.dumps(r) + "\n") | |
| print(f"wrote {args.path}") | |
| return 0 | |
| def cmd_runs(logs: LogStore, args) -> int: | |
| rows = logs.list_runs(args.tester, args.n) | |
| header = ("job", 16), ("kind", 9), ("created (UTC)", 19), ("tester", 12), ("status", 11), ("sim s", 5) | |
| header += (("run s", 6), ("cost $", 6)) | |
| print(" ".join(f"{h:<{w}}" for h, w in header) + " source") | |
| for r in rows: | |
| gpu = f"{r['execution_ms'] / 1000:.1f}" if r.get("execution_ms") else "-" | |
| cost = f"{r['cost_usd']:.3f}" if r.get("cost_usd") is not None else "-" | |
| if r.get("parent_id"): | |
| source = f"run {r['parent_id']}" | |
| else: | |
| source = f"generation {r['generation_id']}" if r.get("generation_id") else "raw input" | |
| print( | |
| f"{r['id']:<16} {(r.get('kind') or 'sim'):<9} {r['created_at'][:19]:<19} {_short(r['tester'], 12):<12} " | |
| f"{r['status']:<11} {r['duration_s'] or 0:<5g} {gpu:<6} {cost:<6} {source}" | |
| ) | |
| if not rows: | |
| print("(no runs)") | |
| return 0 | |
| def cmd_usage(logs: LogStore, args) -> int: | |
| from .jobs import month_spend | |
| settings = get_settings() | |
| rows = logs.usage() | |
| spent, limit = month_spend(settings, logs), settings.monthly_credit_usd | |
| print(f"this month (estimated): ${spent:.2f}" + (f" of ${limit:.2f} limit" if limit else " (no limit)")) | |
| if not rows: | |
| print("(no jobs)") | |
| for u in rows: | |
| cost = f" ~${u['cost_usd']:.2f}" | |
| flag = " <-- burst" if u["max_in_10min"] >= settings.run_burst_warn else "" | |
| print( | |
| f"{u['tester']:<16} runs {u['runs']:>4} last 24h {u['last_24h']:>4} busiest 10 min " | |
| f"{u['max_in_10min']:>3} GPU {u['gpu_seconds']:>7.1f} s{cost} last {u['last'][:19]}{flag}" | |
| ) | |
| return 0 | |
| def main(argv: list[str] | None = None) -> int: | |
| parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) | |
| parser.add_argument("--db", type=Path, help="default: $ROSDIFF_LOG_DB or data/generations.sqlite3") | |
| sub = parser.add_subparsers(dest="cmd", required=True) | |
| sub.add_parser("summary") | |
| p = sub.add_parser("list") | |
| p.add_argument("--kind", choices=["simulate", "code"]) | |
| p.add_argument("--tester") | |
| p.add_argument("--unreviewed", action="store_true") | |
| p.add_argument("-n", type=int, default=50) | |
| p = sub.add_parser("show") | |
| p.add_argument("id") | |
| p.add_argument("--attempts", action="store_true", help="also print every raw model attempt") | |
| p = sub.add_parser("files") | |
| p.add_argument("id") | |
| p.add_argument("dir") | |
| p = sub.add_parser("mark") | |
| p.add_argument("id") | |
| p.add_argument("verdict", choices=["correct", "partial", "incorrect"]) | |
| p.add_argument("--note", default="") | |
| p = sub.add_parser("export") | |
| p.add_argument("path") | |
| p = sub.add_parser("runs") | |
| p.add_argument("--tester") | |
| p.add_argument("-n", type=int, default=50) | |
| sub.add_parser("usage") | |
| args = parser.parse_args(argv) | |
| logs = LogStore(args.db or get_settings().log_db) | |
| handlers = { | |
| "list": cmd_list, | |
| "show": cmd_show, | |
| "files": cmd_files, | |
| "mark": cmd_mark, | |
| "summary": cmd_summary, | |
| "export": cmd_export, | |
| "runs": cmd_runs, | |
| "usage": cmd_usage, | |
| } | |
| return handlers[args.cmd](logs, args) | |
| if __name__ == "__main__": | |
| sys.exit(main()) | |