Create chatcode/evaluate_articles.py

chatcode/evaluate_articles.py

@@ -0,0 +1,47 @@
+"""Cloud-only format/citation evaluation; human factual review is still required."""
+import argparse
+from contextlib import nullcontext
+import json
+from pathlib import Path
+
+from .articles import MODES, behavior_examples, draft_article
+from .runtime import CONFIG, load_engine
+
+
+def evaluate(args):
+    examples = behavior_examples()[args.split]
+    tokenizer, model = load_engine(args.adapter)
+    results = {"base_model": CONFIG["base_model"], "base_revision": CONFIG["base_revision"],
+               "adapter": args.adapter, "split": args.split, "cases_per_variant": len(examples),
+               "human_factual_review_required": True,
+               "limitations": ["Small fictional behavior suite, not a real-world article-quality benchmark.",
+                               "Citation ID validity does not establish source reliability or factual entailment."]}
+    for variant in (["base", "adapter"] if args.adapter else ["base"]):
+        context = model.disable_adapter() if variant == "base" and args.adapter else nullcontext()
+        cases = []
+        with context:
+            for example in examples:
+                request = json.loads(example["messages"][1]["content"])
+                source_rows = [[s["title"], s["url"], s["excerpt"]] for s in request["source_packet"]]
+                expected = json.loads(example["messages"][-1]["content"])
+                mode = MODES[0] if "lead" in expected else MODES[1]
+                case = {"group_id": example["group_id"], "topic": request["topic"], "mode": mode,
+                        "source_packet": request["source_packet"], "human_review": "pending"}
+                try:
+                    rendered, raw = draft_article(tokenizer, model, request["topic"], source_rows, mode)
+                    case.update({"format_and_citation_ids_accepted": True, "output": raw, "rendered": rendered})
+                except ValueError as error:
+                    case.update({"format_and_citation_ids_accepted": False, "error": str(error)})
+                cases.append(case)
+        results[variant] = {"format_acceptance_rate": sum(c["format_and_citation_ids_accepted"] for c in cases) / len(cases),
+                            "cases": cases}
+    Path(args.output).write_text(json.dumps(results, ensure_ascii=False, indent=2), encoding="utf-8")
+    print(f"Wrote {args.output}; source support, tone and usefulness still need human review.")
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="Evaluate article formatting and citation IDs on a cloud GPU")
+    parser.add_argument("--adapter")
+    parser.add_argument("--split", choices=["validation", "test"], default="test")
+    parser.add_argument("--output", required=True)
+    evaluate(parser.parse_args())