| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| """Evaluate a LoRA adapter on InvoiceGuard tasks and upload JSON artifacts.""" |
|
|
| from __future__ import annotations |
|
|
| import argparse |
| import json |
| import os |
| import sys |
| from datetime import datetime, timezone |
| from pathlib import Path |
| from typing import Optional |
|
|
|
|
| def _hf_token() -> Optional[str]: |
| return os.environ.get("HF_TOKEN") or os.environ.get("API_TOKEN_HF") |
|
|
|
|
| def _bootstrap_invoice_guard_path() -> Path: |
| code_dir = os.environ.get("INVOICEGUARD_CODE_DIR") |
| if code_dir and Path(code_dir).is_dir(): |
| sys.path.insert(0, code_dir) |
| return Path(code_dir) |
|
|
| repo = os.environ.get("INVOICEGUARD_CODE_REPO") |
| if repo: |
| from huggingface_hub import snapshot_download |
|
|
| local = snapshot_download(repo_id=repo, repo_type="model", token=_hf_token()) |
| sys.path.insert(0, local) |
| return Path(local) |
|
|
| here = Path(__file__).resolve().parent.parent |
| sys.path.insert(0, str(here)) |
| return here |
|
|
|
|
| _CODE_ROOT = _bootstrap_invoice_guard_path() |
|
|
| import torch |
| from huggingface_hub import HfApi |
| from peft import PeftModel |
| from transformers import AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig |
|
|
| from server.invoice_guard_environment import InvoiceGuardEnvironment |
| from tasks import HARD_TASK_LIST, TASK_LIST |
| from training.rollout import rollout_episode |
|
|
|
|
| def _task_slice(name: str): |
| if name == "canonical": |
| return list(TASK_LIST) |
| if name == "hard": |
| return list(HARD_TASK_LIST) |
| return list(TASK_LIST) + list(HARD_TASK_LIST) |
|
|
|
|
| def main() -> None: |
| p = argparse.ArgumentParser() |
| p.add_argument("--base-model", default=os.environ.get("BASE_MODEL", "Qwen/Qwen3-4B-Instruct-2507")) |
| p.add_argument("--adapter-repo", required=True) |
| p.add_argument("--slice", choices=["canonical", "hard", "all"], default="all") |
| p.add_argument("--max-tasks", type=int, default=None) |
| p.add_argument("--max-new-tokens", type=int, default=384) |
| p.add_argument("--max-prompt-tokens", type=int, default=2048) |
| p.add_argument("--artifact-dir", default="/tmp/invoiceguard-adapter-eval") |
| args = p.parse_args() |
|
|
| token = _hf_token() |
| if not token: |
| raise RuntimeError("HF_TOKEN/API_TOKEN_HF is required for adapter eval upload.") |
|
|
| print(f"[setup] code_root={_CODE_ROOT}", flush=True) |
| print(f"[setup] base_model={args.base_model}", flush=True) |
| print(f"[setup] adapter_repo={args.adapter_repo}", flush=True) |
| print(f"[setup] cuda available={torch.cuda.is_available()}", flush=True) |
|
|
| device = torch.device("cuda" if torch.cuda.is_available() else "cpu") |
| dtype = torch.bfloat16 if torch.cuda.is_available() else torch.float32 |
| quant_cfg = None |
| if torch.cuda.is_available(): |
| quant_cfg = BitsAndBytesConfig( |
| load_in_4bit=True, |
| bnb_4bit_quant_type="nf4", |
| bnb_4bit_use_double_quant=True, |
| bnb_4bit_compute_dtype=dtype, |
| ) |
|
|
| tokenizer = AutoTokenizer.from_pretrained(args.base_model, use_fast=True, token=token) |
| if tokenizer.pad_token is None: |
| tokenizer.pad_token = tokenizer.eos_token |
|
|
| base = AutoModelForCausalLM.from_pretrained( |
| args.base_model, |
| torch_dtype=dtype, |
| device_map="auto" if torch.cuda.is_available() else None, |
| low_cpu_mem_usage=True, |
| quantization_config=quant_cfg, |
| token=token, |
| ) |
| base.config.pad_token_id = tokenizer.pad_token_id |
| base.config.use_cache = False |
| model = PeftModel.from_pretrained(base, args.adapter_repo, token=token) |
| model.eval() |
|
|
| tasks = _task_slice(args.slice) |
| if args.max_tasks is not None: |
| tasks = tasks[: args.max_tasks] |
|
|
| env = InvoiceGuardEnvironment() |
| rows = [] |
| for i, task_id in enumerate(tasks, 1): |
| print(f"[eval] {i}/{len(tasks)} {task_id.value}", flush=True) |
| traj = rollout_episode( |
| model, |
| tokenizer, |
| env, |
| task_id, |
| temperature=0.0001, |
| top_p=1.0, |
| max_new_tokens=args.max_new_tokens, |
| max_prompt_tokens=args.max_prompt_tokens, |
| device=device, |
| ) |
| rows.append({ |
| "task_id": task_id.value, |
| "grader_score": traj.grader_score, |
| "cumulative_reward": traj.cumulative_reward, |
| "success": traj.success, |
| "n_steps": traj.n_steps, |
| "terminal_decision": traj.terminal_decision, |
| "actions": [step.completion_text for step in traj.steps], |
| "step_rewards": [step.reward for step in traj.steps], |
| }) |
|
|
| summary = { |
| "run_finished_at": datetime.now(timezone.utc).isoformat(), |
| "base_model": args.base_model, |
| "adapter_repo": args.adapter_repo, |
| "slice": args.slice, |
| "n_tasks": len(rows), |
| "avg_grader_score": sum(r["grader_score"] for r in rows) / max(len(rows), 1), |
| "avg_cumulative_reward": sum(r["cumulative_reward"] for r in rows) / max(len(rows), 1), |
| "success_rate": sum(1.0 if r["success"] else 0.0 for r in rows) / max(len(rows), 1), |
| "avg_steps": sum(r["n_steps"] for r in rows) / max(len(rows), 1), |
| } |
|
|
| out_dir = Path(args.artifact_dir) |
| out_dir.mkdir(parents=True, exist_ok=True) |
| (out_dir / "adapter_eval_results.json").write_text(json.dumps(rows, indent=2), encoding="utf-8") |
| (out_dir / "adapter_eval_summary.json").write_text(json.dumps(summary, indent=2), encoding="utf-8") |
| print(json.dumps(summary, indent=2), flush=True) |
|
|
| HfApi(token=token).upload_folder( |
| folder_path=str(out_dir), |
| repo_id=args.adapter_repo, |
| repo_type="model", |
| path_in_repo=f"eval_artifacts/{args.slice}", |
| token=token, |
| commit_message=f"Add InvoiceGuard adapter eval results ({args.slice})", |
| ) |
| print(f"[push] eval artifacts uploaded to https://huggingface.co/{args.adapter_repo}", flush=True) |
|
|
|
|
| if __name__ == "__main__": |
| main() |
|
|
