"""Cloud-only format/citation evaluation; human factual review is still required."""
import argparse
from contextlib import nullcontext
import json
from pathlib import Path

from .articles import MODES, behavior_examples, draft_article
from .runtime import CONFIG, load_engine


def evaluate(args):
    examples = behavior_examples()[args.split]
    tokenizer, model = load_engine(args.adapter)
    results = {"base_model": CONFIG["base_model"], "base_revision": CONFIG["base_revision"],
               "adapter": args.adapter, "split": args.split, "cases_per_variant": len(examples),
               "human_factual_review_required": True,
               "limitations": ["Small fictional behavior suite, not a real-world article-quality benchmark.",
                               "Citation ID validity does not establish source reliability or factual entailment."]}
    for variant in (["base", "adapter"] if args.adapter else ["base"]):
        context = model.disable_adapter() if variant == "base" and args.adapter else nullcontext()
        cases = []
        with context:
            for example in examples:
                request = json.loads(example["messages"][1]["content"])
                source_rows = [[s["title"], s["url"], s["excerpt"]] for s in request["source_packet"]]
                expected = json.loads(example["messages"][-1]["content"])
                mode = MODES[0] if "lead" in expected else MODES[1]
                case = {"group_id": example["group_id"], "topic": request["topic"], "mode": mode,
                        "source_packet": request["source_packet"], "human_review": "pending"}
                try:
                    rendered, raw = draft_article(tokenizer, model, request["topic"], source_rows, mode)
                    case.update({"format_and_citation_ids_accepted": True, "output": raw, "rendered": rendered})
                except ValueError as error:
                    case.update({"format_and_citation_ids_accepted": False, "error": str(error)})
                cases.append(case)
        results[variant] = {"format_acceptance_rate": sum(c["format_and_citation_ids_accepted"] for c in cases) / len(cases),
                            "cases": cases}
    Path(args.output).write_text(json.dumps(results, ensure_ascii=False, indent=2), encoding="utf-8")
    print(f"Wrote {args.output}; source support, tone and usefulness still need human review.")


if __name__ == "__main__":
    parser = argparse.ArgumentParser(description="Evaluate article formatting and citation IDs on a cloud GPU")
    parser.add_argument("--adapter")
    parser.add_argument("--split", choices=["validation", "test"], default="test")
    parser.add_argument("--output", required=True)
    evaluate(parser.parse_args())