api/review.py

"""Inspect and grade what testers generated.

python -m api.review summary                      counts per kind: auto-valid, tester-rated, reviewed
python -m api.review list [--kind simulate|code] [--tester NAME] [--unreviewed] [-n 50]
python -m api.review show ID [--attempts]         request, retrieved sources, output, validation
python -m api.review files ID DIR                 write the output to DIR (scene.xml or the package files)
python -m api.review mark ID correct|partial|incorrect [--note TEXT]
python -m api.review export out.jsonl             everything, one JSON object per line
python -m api.review runs [--tester NAME] [-n 50] every /simulate/run call: who, when, status, GPU time
python -m api.review usage                        runs per tester: total, last 24 h, busiest 10 minutes
"""

from __future__ import annotations

import argparse
import json
import sys
from pathlib import Path

from .config import get_settings
from .logstore import LogStore


def _short(text: str, n: int = 60) -> str:
    text = " ".join(str(text).split())
    return text if len(text) <= n else text[: n - 1] + "…"


def _request_text(row: dict) -> str:
    req = row["request"] or {}
    return req.get("description") or req.get("request") or ""


def _auto(row: dict) -> str:
    if row.get("error"):
        return "ERROR"
    return {True: "valid", False: "invalid", None: "-"}[row.get("valid")]


def cmd_list(logs: LogStore, args) -> int:
    rows = logs.list(args.kind, args.tester, args.unreviewed, args.n)
    header = ("id", 12), ("created (UTC)", 19), ("tester", 10), ("kind", 8), ("auto", 7), ("tester", 7), ("review", 9)
    print("  ".join(f"{h:<{w}}" for h, w in header) + "  request")
    for r in rows:
        print(
            f"{r['id']:<12}  {r['created_at'][:19]:<19}  {_short(r['tester'], 10):<10} {r['kind']:<8} "
            f"{_auto(r):<7} {r['tester_rating'] or '-':<7} {r['review_verdict'] or '-':<9} {_short(_request_text(r))}"
        )
    if not rows:
        print("(no generations)")
    return 0


def cmd_show(logs: LogStore, args) -> int:
    r = logs.get(args.id)
    if not r:
        print(f"no generation {args.id}", file=sys.stderr)
        return 1
    print(f"id:        {r['id']}   kind: {r['kind']}   tester: {r['tester']}   at: {r['created_at']}")
    print(f"model:     {r['model']}   duration: {r['duration_ms']} ms   attempts: {len(r['attempts'] or [])}")
    print(f"request:   {json.dumps(r['request'])}")
    print(f"auto:      {_auto(r)}  {json.dumps(r['validation'])}")
    if r.get("error"):
        print(f"error:     {r['error']}")
    print(f"tester:    {r['tester_rating'] or '-'}  {r['tester_note'] or ''}")
    print(f"review:    {r['review_verdict'] or '-'}  {r['review_note'] or ''}")
    print("sources:")
    for s in r["sources"] or []:
        print(f"  - {s}")
    if args.attempts:
        for i, a in enumerate(r["attempts"] or [], 1):
            print(f"\n----- attempt {i}: {json.dumps(a.get('validation'))}")
            print(a.get("raw_output", ""))
    print("\n----- output")
    if r["kind"] == "code" and r["output"]:
        for path, content in json.loads(r["output"]).items():
            print(f"\n=== {path}\n{content}")
    else:
        print(r["output"] or "(none)")
    return 0


def cmd_files(logs: LogStore, args) -> int:
    r = logs.get(args.id)
    if not r or not r["output"]:
        print(f"no output stored for {args.id}", file=sys.stderr)
        return 1
    out = Path(args.dir)
    out.mkdir(parents=True, exist_ok=True)
    files = json.loads(r["output"]) if r["kind"] == "code" else {"scene_rosdiff.xml": r["output"]}
    for path, content in files.items():
        dest = (out / path).resolve()
        if out.resolve() not in dest.parents:
            print(f"skipping unsafe path {path}", file=sys.stderr)
            continue
        dest.parent.mkdir(parents=True, exist_ok=True)
        dest.write_text(content)
        print(f"wrote {dest}")
    return 0


def cmd_mark(logs: LogStore, args) -> int:
    if not logs.set_review(args.id, args.verdict, args.note):
        print(f"no generation {args.id}", file=sys.stderr)
        return 1
    print(f"{args.id}: {args.verdict}")
    return 0


def cmd_summary(logs: LogStore, args) -> int:
    rows = logs.summary()
    if not rows:
        print("(no generations)")
    for r in rows:
        print(
            f"{r['kind']:<9} total {r['total']:>4} | auto-valid {r['auto_valid'] or 0:>4} | "
            f"tester says works {r['tester_works'] or 0}/{r['tester_rated'] or 0} | "
            f"reviewed correct {r['reviewed_correct'] or 0}/{r['reviewed'] or 0}"
        )
    return 0


def cmd_export(logs: LogStore, args) -> int:
    with open(args.path, "w") as f:
        for r in logs.all():
            f.write(json.dumps(r) + "\n")
    print(f"wrote {args.path}")
    return 0


def cmd_runs(logs: LogStore, args) -> int:
    rows = logs.list_runs(args.tester, args.n)
    header = ("job", 16), ("kind", 9), ("created (UTC)", 19), ("tester", 12), ("status", 11), ("sim s", 5)
    header += (("run s", 6), ("cost $", 6))
    print("  ".join(f"{h:<{w}}" for h, w in header) + "  source")
    for r in rows:
        gpu = f"{r['execution_ms'] / 1000:.1f}" if r.get("execution_ms") else "-"
        cost = f"{r['cost_usd']:.3f}" if r.get("cost_usd") is not None else "-"
        if r.get("parent_id"):
            source = f"run {r['parent_id']}"
        else:
            source = f"generation {r['generation_id']}" if r.get("generation_id") else "raw input"
        print(
            f"{r['id']:<16}  {(r.get('kind') or 'sim'):<9}  {r['created_at'][:19]:<19}  {_short(r['tester'], 12):<12}  "
            f"{r['status']:<11}  {r['duration_s'] or 0:<5g}  {gpu:<6}  {cost:<6}  {source}"
        )
    if not rows:
        print("(no runs)")
    return 0


def cmd_usage(logs: LogStore, args) -> int:
    from .jobs import month_spend

    settings = get_settings()
    rows = logs.usage()
    spent, limit = month_spend(settings, logs), settings.monthly_credit_usd
    print(f"this month (estimated): ${spent:.2f}" + (f" of ${limit:.2f} limit" if limit else " (no limit)"))
    if not rows:
        print("(no jobs)")
    for u in rows:
        cost = f" ~${u['cost_usd']:.2f}"
        flag = "  <-- burst" if u["max_in_10min"] >= settings.run_burst_warn else ""
        print(
            f"{u['tester']:<16} runs {u['runs']:>4}  last 24h {u['last_24h']:>4}  busiest 10 min "
            f"{u['max_in_10min']:>3}  GPU {u['gpu_seconds']:>7.1f} s{cost}  last {u['last'][:19]}{flag}"
        )
    return 0


def main(argv: list[str] | None = None) -> int:
    parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
    parser.add_argument("--db", type=Path, help="default: $ROSDIFF_LOG_DB or data/generations.sqlite3")
    sub = parser.add_subparsers(dest="cmd", required=True)
    sub.add_parser("summary")
    p = sub.add_parser("list")
    p.add_argument("--kind", choices=["simulate", "code"])
    p.add_argument("--tester")
    p.add_argument("--unreviewed", action="store_true")
    p.add_argument("-n", type=int, default=50)
    p = sub.add_parser("show")
    p.add_argument("id")
    p.add_argument("--attempts", action="store_true", help="also print every raw model attempt")
    p = sub.add_parser("files")
    p.add_argument("id")
    p.add_argument("dir")
    p = sub.add_parser("mark")
    p.add_argument("id")
    p.add_argument("verdict", choices=["correct", "partial", "incorrect"])
    p.add_argument("--note", default="")
    p = sub.add_parser("export")
    p.add_argument("path")
    p = sub.add_parser("runs")
    p.add_argument("--tester")
    p.add_argument("-n", type=int, default=50)
    sub.add_parser("usage")
    args = parser.parse_args(argv)
    logs = LogStore(args.db or get_settings().log_db)
    handlers = {
        "list": cmd_list,
        "show": cmd_show,
        "files": cmd_files,
        "mark": cmd_mark,
        "summary": cmd_summary,
        "export": cmd_export,
        "runs": cmd_runs,
        "usage": cmd_usage,
    }
    return handlers[args.cmd](logs, args)


if __name__ == "__main__":
    sys.exit(main())