Spaces:
Running on Zero
Running on Zero
Download scripts/critic_supervise.py from Kagzy/ResearchOS: direct link, hf CLI and curl.
- Browser
- Download file 4.89 kB
-
https://huggingface.co/spaces/Kagzy/ResearchOS/resolve/main/scripts/critic_supervise.py
- Command line
-
hf download hf://spaces/Kagzy/ResearchOS/scripts/critic_supervise.py
-
curl -L -o critic_supervise.py https://huggingface.co/spaces/Kagzy/ResearchOS/resolve/main/scripts/critic_supervise.py
4.89 kB
| """Stage 2: build reference critiques for the trajectories (Algorithm 2). | |
| python scripts/critic_supervise.py --limit 300 -k 3 | |
| Reference verdicts come from HotpotQA gold answers, so this script only spends | |
| API budget where the teacher is actually needed: the location/reason/fix | |
| fields of trajectories that went wrong. CORRECT trajectories are labelled for | |
| free, because Figure 7(b) fixes their form to location="none", fix="keep". | |
| Budget-aware by construction: Groq's free tier allows 1,000 requests/day, so | |
| the script tracks spend on disk, stops cleanly when the quota is gone, and | |
| resumes where it left off on the next run. | |
| """ | |
| from __future__ import annotations | |
| import argparse | |
| import json | |
| import sys | |
| import time | |
| from pathlib import Path | |
| sys.path.insert(0, str(Path(__file__).resolve().parents[1])) | |
| if hasattr(sys.stdout, "reconfigure"): | |
| sys.stdout.reconfigure(encoding="utf-8", errors="replace") | |
| import src.config # noqa: F401,E402 - loads .env | |
| from src.critic.supervise import ( # noqa: E402 | |
| Budget, BudgetExhausted, DEFAULT_TEACHER, SupervisedCritique, Teacher, | |
| diagnose_trajectory, read_supervision, supervise_trajectory, | |
| supervision_summary, write_supervision, | |
| ) | |
| from src.critic.trajectory import read_trajectories # noqa: E402 | |
| def main() -> None: | |
| ap = argparse.ArgumentParser(description=__doc__) | |
| ap.add_argument("--trajectories", default="data/critic/trajectories_train.jsonl") | |
| ap.add_argument("--out", default="data/critic/supervision_train.jsonl") | |
| ap.add_argument("--budget-file", default="data/critic/.groq_budget.json") | |
| ap.add_argument("--limit", type=int, default=None, help="max trajectories to label") | |
| ap.add_argument("-k", "--samples", type=int, default=3, | |
| help="teacher critiques per trajectory (Algorithm 2 K)") | |
| ap.add_argument("--model", default=DEFAULT_TEACHER) | |
| ap.add_argument("--max-requests", type=int, default=None, | |
| help="cap this run's requests (default: whatever today has left)") | |
| ap.add_argument("--mode", default="diagnose", choices=["diagnose", "consensus"], | |
| help="diagnose: verdict is given, teacher only localises " | |
| "(every request yields a label). consensus: Algorithm 2 " | |
| "as published, where the teacher also votes on the verdict.") | |
| args = ap.parse_args() | |
| trajectories = read_trajectories(args.trajectories) | |
| out = Path(args.out) | |
| done = {s.qid: s for s in read_supervision(out)} if out.exists() else {} | |
| todo = [t for t in trajectories if t.qid not in done] | |
| if args.limit: | |
| todo = todo[: max(0, args.limit - len(done))] | |
| budget = Budget.load(args.budget_file) | |
| if args.max_requests is not None: | |
| budget.limit = min(budget.limit, budget.requests + args.max_requests) | |
| # Only INCORRECT/UNSURE trajectories cost requests. | |
| needs_teacher = [t for t in todo if t.verdict.value != "CORRECT"] | |
| print(f"[sup] {len(trajectories)} trajectories, {len(done)} already labelled") | |
| print(f"[sup] {len(todo)} to do; {len(needs_teacher)} need the teacher " | |
| f"({len(todo) - len(needs_teacher)} derived free)") | |
| print(f"[sup] budget today: {budget.requests}/{budget.limit} used, " | |
| f"{budget.remaining} left -> about {budget.remaining // max(args.samples,1)} " | |
| f"teacher-labelled trajectories at k={args.samples}") | |
| teacher = Teacher(args.model, budget=budget) | |
| results: list[SupervisedCritique] = list(done.values()) | |
| started, labelled, stopped = time.time(), 0, None | |
| for i, traj in enumerate(todo, 1): | |
| try: | |
| label = (diagnose_trajectory if args.mode == 'diagnose' | |
| else supervise_trajectory) | |
| results.append(label(traj, teacher, k=args.samples)) | |
| labelled += 1 | |
| except BudgetExhausted as exc: | |
| stopped = str(exc) | |
| break | |
| except Exception as exc: # noqa: BLE001 - never lose completed work | |
| print(f"[sup] {traj.qid}: {type(exc).__name__}: {str(exc)[:90]}") | |
| continue | |
| if labelled % 10 == 0 or i == len(todo): | |
| write_supervision(out, results) # checkpoint | |
| rate = labelled / max(time.time() - started, 1e-6) | |
| print(f"[sup] {len(results)} labelled ({rate*60:.1f}/min) " | |
| f"budget {budget.requests}/{budget.limit}", flush=True) | |
| write_supervision(out, results) | |
| summary = supervision_summary(results) | |
| print(f"\n[sup] wrote {len(results)} reference critiques to {out}") | |
| print(json.dumps(summary, indent=2)) | |
| if stopped: | |
| print(f"\n[sup] STOPPED: {stopped}") | |
| print("[sup] re-run tomorrow to continue; completed work is saved.") | |
| if __name__ == "__main__": | |
| main() | |