import json, os, sys, hashlib
sys.path.insert(0, os.path.join(os.environ.get("GPT2EVAL_PKG", "package"), "code")); import stasis_gpt2_eval as E
O = "out"; L = lambda f: [json.loads(l) for l in open(os.path.join(O, f))]
rs = {b: L(f"ROWS_{b}_STASIS_36420.jsonl") for b in E.BENCHES}; rg = {b: L(f"ROWS_{b}_GPT2_SMALL.jsonl") for b in E.BENCHES}
S = json.loads(json.dumps(E.summarize(rs, rg)).replace("STASIS_12269", "STASIS_36420"))
# reproducibility vs Kaggle T4 GPT-2 rows
rep = {}
for b in E.BENCHES:
    k = [json.loads(l) for l in open(f"step12269_rows/ROWS_{b}_GPT2_SMALL.jsonl")]; c = rg[b]
    if "nll" in c[0]: rep[b] = {"max_rel_nll_diff": max(abs(a["nll"]-x["nll"])/max(1,abs(x["nll"])) for a,x in zip(c,k)), "greedy_diffs": sum(a["greedy"]!=x["greedy"] for a,x in zip(c,k))}
    else: rep[b] = {"acc_diffs": sum(a["acc"]!=x["acc"] for a,x in zip(c,k)), "acc_norm_diffs": sum(a["acc_norm"]!=x["acc_norm"] for a,x in zip(c,k))}
import torch, platform
R = {"label": "REAL", "metric_definitions_sha256": E.METRIC_SHA, "checkpoint": {"file": "endpoint.model_only.pt", "step": 36420, "sha256": "5b2dc755ed6e38817be84aa982e4591f64c8cc94ce1fa0316c2ac0336e0bcf85"},
     "env": {"torch": torch.__version__, "python": platform.python_version(), "device": "cpu (8 threads, fp32)", "tok_budget": 8192, "limit": None},
     "note": "Same package items, tokenization, scoring code and bootstrap as the step-12,269 Kaggle eval; both models re-scored on box CPU. GPT-2 CPU rows vs step-12,269 Kaggle T4 GPT-2 rows: see gpt2_reproduction.",
     "gpt2_reproduction": rep, "results": S,
     "previous_step12269": json.load(open("step12269_rows/RESULTS.json"))["results"]}
json.dump(R, open(os.path.join(O, "RESULTS.json"), "w"), indent=1); print(json.dumps(S, indent=0)[:200]); print(rep)