# /// script # dependencies = ["torch==2.8.0", "transformers==4.56.2", "huggingface_hub==0.34.4", "accelerate==1.10.1", "safetensors==0.6.2"] # /// """Independent GSM8K and HumanEval subset comparison on remote GPUs.""" import json,re,sys,subprocess,tempfile,ast,urllib.request,urllib.parse,gc,hashlib from pathlib import Path import torch from transformers import AutoModelForCausalLM,AutoTokenizer from huggingface_hub import HfApi,hf_hub_download,CommitOperationAdd def task_check(case,text): final=text.split("")[-1] numbers=re.findall(r"-?\d+(?:\.\d+)?",final.replace(",","")) return bool(numbers) and abs(float(numbers[-1])-float(case["expected"]))<1e-6 api=HfApi();REPO="SEE7/Astheral";BASE="NousResearch/DeepHermes-3-Llama-3-8B-Preview";REV="53d902c7dcfa4c749a3b455149064f82af2e2549" ROOT=Path(__file__).resolve().parent BRANCH="v3-candidate-20261113" revision=api.model_info(REPO,revision=BRANCH).sha settings=json.loads(Path(hf_hub_download(REPO,"settings.json",revision=revision)).read_text()) def rows(repo,config,split,offset,length): q=urllib.parse.urlencode(dict(dataset=repo,config=config,split=split,offset=offset,length=length)) return [r["row"] for r in json.load(urllib.request.urlopen("https://datasets-server.huggingface.co/rows?"+q,timeout=60))["rows"]] math_revision=api.dataset_info("openai/gsm8k").sha code_revision=api.dataset_info("openai/openai_humaneval").sha cases=[] for i,r in enumerate(rows("openai/gsm8k","main","test",231,8)): expected=float(r["answer"].rsplit("####",1)[1].strip().replace(",","")) cases.append({"id":"gsm8k-test-"+str(231+i),"mode":"thinking","prompt":r["question"]+"\nGive the final numerical answer after your explanation.","kind":"number","expected":expected,"source":"openai/gsm8k","source_revision":math_revision}) chosen={"HumanEval/"+str(i) for i in [0,3,7,12,18,25,36,45]} for r in rows("openai/openai_humaneval","openai_humaneval","test",0,50): if r["task_id"] not in chosen:continue cases.append({"id":r["task_id"],"mode":"instruct","prompt":"Return only the complete Python implementation, including necessary imports, for the following function specification:\n\n"+r["prompt"],"kind":"code","entry_point":r["entry_point"],"tests":r["test"]+"\ncheck("+r["entry_point"]+")","source":"openai/openai_humaneval","source_revision":code_revision}) assert len(cases)==16 def check_code(c,text): blocks=re.findall(r"\x60\x60\x60(?:python)?\s*\n(.*?)\x60\x60\x60",text,re.S) code=blocks[0] if blocks else text.strip() try: tree=ast.parse(code) for n in ast.walk(tree): if isinstance(n,(ast.Import,ast.ImportFrom)): names=[a.name.split(".")[0] for a in n.names] if isinstance(n,ast.Import) else [(n.module or "").split(".")[0]] if any(x not in ["math","collections","itertools","functools","re","typing","statistics","heapq","bisect","string"] for x in names):return False if isinstance(n,ast.Name) and n.id in ["open","exec","eval","compile","__import__","__builtins__","globals","locals","getattr","setattr","input"]:return False if isinstance(n,ast.Attribute) and n.attr.startswith("__"):return False with tempfile.TemporaryDirectory() as d: p=Path(d)/"check.py" p.write_text("import resource\nresource.setrlimit(resource.RLIMIT_CPU,(3,3))\nresource.setrlimit(resource.RLIMIT_AS,(536870912,536870912))\n"+code+"\n"+c["tests"]) r=subprocess.run([sys.executable,"-I",str(p)],cwd=d,timeout=5,capture_output=True,env={"PATH":"/usr/bin:/bin"}) return r.returncode==0 except Exception:return False all_results={};scores={};paired_results={} source=Path(hf_hub_download(REPO,'v3/train_v3.py',revision=revision)).read_text() grade_node=next(n for n in ast.parse(source).body if isinstance(n,ast.FunctionDef) and n.name=='grade') grade_scope={'json':json,'re':re};exec(compile(ast.Module(body=[grade_node],type_ignores=[]),'grader','exec'),grade_scope) paired_cases=json.loads(Path(hf_hub_download(REPO,'v3-results/heldout_cases.json',revision=revision)).read_text()) for tag,repo,rev in [("base",REPO,"d9aa899722be4788f57107c2951c72d0ec6614ee"),("astheral",REPO,revision)]: tok=AutoTokenizer.from_pretrained(repo,revision=rev);tok.pad_token_id=128004;tok.padding_side="left" model=AutoModelForCausalLM.from_pretrained(repo,revision=rev,torch_dtype=torch.bfloat16,device_map={"":"cuda:0"},attn_implementation="sdpa").eval() paired=[] with torch.inference_mode(): for c in paired_cases: inp=tok.apply_chat_template([{"role":"system","content":settings["system"]},{"role":"user","content":c["prompt"]}],tokenize=True,add_generation_prompt=True,return_dict=True,return_tensors="pt").to("cuda") output=model.generate(**inp,max_new_tokens=96,do_sample=False,pad_token_id=128004,eos_token_id=128009) text=tok.decode(output[0,inp["input_ids"].shape[1]:],skip_special_tokens=True) paired.append({**c,"output":text,"pass":grade_scope["grade"](c,text)}) print("BF16_PAIRED",tag,c["id"],paired[-1]["pass"],flush=True) paired_results[tag]=paired result=[] with torch.inference_mode(): for mode in ["instruct","thinking"]: selected=[c for c in cases if c["mode"]==mode] for i in range(0,len(selected),2): group=selected[i:i+2] system=settings["system"]+("\n"+settings["thinking"] if mode=="thinking" else "") texts=[tok.apply_chat_template([{"role":"system","content":system},{"role":"user","content":c["prompt"]}],tokenize=False,add_generation_prompt=True) for c in group] inputs=tok(texts,padding=True,return_tensors="pt").to("cuda") out=model.generate(**inputs,max_new_tokens=1536 if mode=="thinking" else 768,do_sample=False,pad_token_id=128004,eos_token_id=128009) for c,o in zip(group,out): ids=o[inputs["input_ids"].shape[1]:] text=tok.decode(ids,skip_special_tokens=True) passed=check_code(c,text) if mode=="instruct" else task_check(c,text) result.append({**c,"output":text,"pass":passed,"hit_token_limit":len(ids)>=(1536 if mode=="thinking" else 768) and 128009 not in ids.tolist()}) print("HOLDOUT",tag,c["id"],passed,flush=True) all_results[tag]=result scores[tag]={} for kind in ["code","number"]: chosen_rows=[r for r in result if r["kind"]==kind] scores[tag][kind]={"passed":sum(r["pass"] for r in chosen_rows),"total":len(chosen_rows),"token_limited":sum(r["hit_token_limit"] for r in chosen_rows)} del model;gc.collect();torch.cuda.empty_cache() passed=sum(v["passed"] for v in scores["astheral"].values())>=sum(v["passed"] for v in scores["base"].values()) and all(scores["astheral"][k]["passed"]>=scores["base"][k]["passed"] for k in ["code","number"]) passed=passed and sum(r["pass"] for r in paired_results["astheral"])>=sum(r["pass"] for r in paired_results["base"]) report={"comparison":"V2 versus V3, both BF16","v2_revision":"d9aa899722be4788f57107c2951c72d0ec6614ee","paired_bf16_scores":{k:sum(r["pass"] for r in v) for k,v in paired_results.items()},"suite":"GSM8K test rows 231-238; HumanEval selected tasks 0,3,7,12,18,25,36,45","scores":scores,"no_material_regression_gate":passed,"astheral_revision":revision,"settings":{"sampling":False,"code_max_new_tokens":768,"math_max_new_tokens":1536},"hardware":torch.cuda.get_device_name(0),"source_revisions":{"gsm8k":math_revision,"humaneval":code_revision}} changed={"paired_bf16_outputs.json":json.dumps(paired_results,indent=2),"holdout_evaluation.json":json.dumps(report,indent=2),"holdout_outputs.json":json.dumps(all_results,indent=2),"holdout_eval.py":Path(__file__).read_text(),"HUMANEVAL_LICENSE.txt":urllib.request.urlopen("https://raw.githubusercontent.com/openai/human-eval/master/LICENSE",timeout=60).read().decode()} manifest=json.loads(Path(hf_hub_download(REPO,"artifact_manifest.json")).read_text()) for n,b in changed.items():manifest[n]={"bytes":len(b.encode()),"sha256":hashlib.sha256(b.encode()).hexdigest()} changed["artifact_manifest.json"]=json.dumps(manifest,indent=2) api.create_commit(repo_id=REPO,revision=BRANCH,operations=[CommitOperationAdd(path_in_repo=n,path_or_fileobj=b.encode()) for n,b in changed.items()],commit_message="Record measured V2 versus V3 GSM8K and HumanEval comparison") print("HOLDOUT_REPORT",json.dumps(report),flush=True) if not passed:raise SystemExit(5)