File size: 2,754 Bytes
60041df | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 | """Cloud-only format/citation evaluation; human factual review is still required."""
import argparse
from contextlib import nullcontext
import json
from pathlib import Path
from .articles import MODES, behavior_examples, draft_article
from .runtime import CONFIG, load_engine
def evaluate(args):
examples = behavior_examples()[args.split]
tokenizer, model = load_engine(args.adapter)
results = {"base_model": CONFIG["base_model"], "base_revision": CONFIG["base_revision"],
"adapter": args.adapter, "split": args.split, "cases_per_variant": len(examples),
"human_factual_review_required": True,
"limitations": ["Small fictional behavior suite, not a real-world article-quality benchmark.",
"Citation ID validity does not establish source reliability or factual entailment."]}
for variant in (["base", "adapter"] if args.adapter else ["base"]):
context = model.disable_adapter() if variant == "base" and args.adapter else nullcontext()
cases = []
with context:
for example in examples:
request = json.loads(example["messages"][1]["content"])
source_rows = [[s["title"], s["url"], s["excerpt"]] for s in request["source_packet"]]
expected = json.loads(example["messages"][-1]["content"])
mode = MODES[0] if "lead" in expected else MODES[1]
case = {"group_id": example["group_id"], "topic": request["topic"], "mode": mode,
"source_packet": request["source_packet"], "human_review": "pending"}
try:
rendered, raw = draft_article(tokenizer, model, request["topic"], source_rows, mode)
case.update({"format_and_citation_ids_accepted": True, "output": raw, "rendered": rendered})
except ValueError as error:
case.update({"format_and_citation_ids_accepted": False, "error": str(error)})
cases.append(case)
results[variant] = {"format_acceptance_rate": sum(c["format_and_citation_ids_accepted"] for c in cases) / len(cases),
"cases": cases}
Path(args.output).write_text(json.dumps(results, ensure_ascii=False, indent=2), encoding="utf-8")
print(f"Wrote {args.output}; source support, tone and usefulness still need human review.")
if __name__ == "__main__":
parser = argparse.ArgumentParser(description="Evaluate article formatting and citation IDs on a cloud GPU")
parser.add_argument("--adapter")
parser.add_argument("--split", choices=["validation", "test"], default="test")
parser.add_argument("--output", required=True)
evaluate(parser.parse_args())
|