| import json, os, sys, hashlib |
| sys.path.insert(0, os.path.join(os.environ.get("GPT2EVAL_PKG", "package"), "code")); import stasis_gpt2_eval as E |
| O = "out"; L = lambda f: [json.loads(l) for l in open(os.path.join(O, f))] |
| rs = {b: L(f"ROWS_{b}_STASIS_36420.jsonl") for b in E.BENCHES}; rg = {b: L(f"ROWS_{b}_GPT2_SMALL.jsonl") for b in E.BENCHES} |
| S = json.loads(json.dumps(E.summarize(rs, rg)).replace("STASIS_12269", "STASIS_36420")) |
| |
| rep = {} |
| for b in E.BENCHES: |
| k = [json.loads(l) for l in open(f"step12269_rows/ROWS_{b}_GPT2_SMALL.jsonl")]; c = rg[b] |
| if "nll" in c[0]: rep[b] = {"max_rel_nll_diff": max(abs(a["nll"]-x["nll"])/max(1,abs(x["nll"])) for a,x in zip(c,k)), "greedy_diffs": sum(a["greedy"]!=x["greedy"] for a,x in zip(c,k))} |
| else: rep[b] = {"acc_diffs": sum(a["acc"]!=x["acc"] for a,x in zip(c,k)), "acc_norm_diffs": sum(a["acc_norm"]!=x["acc_norm"] for a,x in zip(c,k))} |
| import torch, platform |
| R = {"label": "REAL", "metric_definitions_sha256": E.METRIC_SHA, "checkpoint": {"file": "endpoint.model_only.pt", "step": 36420, "sha256": "5b2dc755ed6e38817be84aa982e4591f64c8cc94ce1fa0316c2ac0336e0bcf85"}, |
| "env": {"torch": torch.__version__, "python": platform.python_version(), "device": "cpu (8 threads, fp32)", "tok_budget": 8192, "limit": None}, |
| "note": "Same package items, tokenization, scoring code and bootstrap as the step-12,269 Kaggle eval; both models re-scored on box CPU. GPT-2 CPU rows vs step-12,269 Kaggle T4 GPT-2 rows: see gpt2_reproduction.", |
| "gpt2_reproduction": rep, "results": S, |
| "previous_step12269": json.load(open("step12269_rows/RESULTS.json"))["results"]} |
| json.dump(R, open(os.path.join(O, "RESULTS.json"), "w"), indent=1); print(json.dumps(S, indent=0)[:200]); print(rep) |
|
|
