| """Cloud-only format/citation evaluation; human factual review is still required.""" |
| import argparse |
| from contextlib import nullcontext |
| import json |
| from pathlib import Path |
|
|
| from .articles import MODES, behavior_examples, draft_article |
| from .runtime import CONFIG, load_engine |
|
|
|
|
| def evaluate(args): |
| examples = behavior_examples()[args.split] |
| tokenizer, model = load_engine(args.adapter) |
| results = {"base_model": CONFIG["base_model"], "base_revision": CONFIG["base_revision"], |
| "adapter": args.adapter, "split": args.split, "cases_per_variant": len(examples), |
| "human_factual_review_required": True, |
| "limitations": ["Small fictional behavior suite, not a real-world article-quality benchmark.", |
| "Citation ID validity does not establish source reliability or factual entailment."]} |
| for variant in (["base", "adapter"] if args.adapter else ["base"]): |
| context = model.disable_adapter() if variant == "base" and args.adapter else nullcontext() |
| cases = [] |
| with context: |
| for example in examples: |
| request = json.loads(example["messages"][1]["content"]) |
| source_rows = [[s["title"], s["url"], s["excerpt"]] for s in request["source_packet"]] |
| expected = json.loads(example["messages"][-1]["content"]) |
| mode = MODES[0] if "lead" in expected else MODES[1] |
| case = {"group_id": example["group_id"], "topic": request["topic"], "mode": mode, |
| "source_packet": request["source_packet"], "human_review": "pending"} |
| try: |
| rendered, raw = draft_article(tokenizer, model, request["topic"], source_rows, mode) |
| case.update({"format_and_citation_ids_accepted": True, "output": raw, "rendered": rendered}) |
| except ValueError as error: |
| case.update({"format_and_citation_ids_accepted": False, "error": str(error)}) |
| cases.append(case) |
| results[variant] = {"format_acceptance_rate": sum(c["format_and_citation_ids_accepted"] for c in cases) / len(cases), |
| "cases": cases} |
| Path(args.output).write_text(json.dumps(results, ensure_ascii=False, indent=2), encoding="utf-8") |
| print(f"Wrote {args.output}; source support, tone and usefulness still need human review.") |
|
|
|
|
| if __name__ == "__main__": |
| parser = argparse.ArgumentParser(description="Evaluate article formatting and citation IDs on a cloud GPU") |
| parser.add_argument("--adapter") |
| parser.add_argument("--split", choices=["validation", "test"], default="test") |
| parser.add_argument("--output", required=True) |
| evaluate(parser.parse_args()) |
|
|
