| """Apply the corrected answer parser to both models' saved generations.""" |
| import json,hashlib |
| from pathlib import Path |
| from huggingface_hub import HfApi,hf_hub_download,CommitOperationAdd |
| from evaluate import check,summary |
| api=HfApi();repo="SEE7/Astheral" |
| def read(name):return Path(hf_hub_download(repo,name)).read_text() |
| m=json.loads(read("evaluation.json")) |
| assert m["version"]=="2" |
| changed={} |
| outputs={} |
| for filename in ["base_outputs.json","astheral_outputs.json","astheral_merged_outputs.json","merged_regressions_outputs.json"]: |
| rows=json.loads(read(filename)) |
| for row in rows:row["pass"]=check(row,row["output"]) |
| outputs[filename]=rows |
| changed[filename]=json.dumps(rows,indent=2) |
| m["base_scores"]=summary(outputs["base_outputs.json"]) |
| m["astheral_scores"]=summary(outputs["astheral_outputs.json"]) |
| m["merged_scores"]=summary(outputs["astheral_merged_outputs.json"]) |
| for field,filename in [("astheral_scores","astheral_outputs.json"),("merged_scores","astheral_merged_outputs.json")]: |
| confirmations=[r for r in outputs[filename] if r["group"]=="confirmation"] |
| if confirmations:m[field]["confirmation"]={"passed":sum(r["pass"] for r in confirmations),"total":len(confirmations)} |
| s=m["merged_scores"];b=m["base_scores"] |
| m["evaluation_parser_revision"]="2: boxed answer, unambiguous leading answer paragraph, then numeric fallback" |
| m["quality_gate_passed"]=(s["regression"]["passed"]==2 and s["fresh"]["passed"]==s["fresh"]["total"] and s["number"]["passed"]>=b["number"]["passed"]) |
| if "confirmation" in s:m["quality_gate_passed"] &= s["confirmation"]["passed"]==s["confirmation"]["total"] |
| mode_rows=outputs["astheral_merged_outputs.json"] |
| mode_passed=sum(("<think>" not in r["output"] and "</think>" not in r["output"]) if r["mode"]=="instruct" else (r["output"].lstrip().startswith("<think>") and "</think>" in r["output"]) for r in mode_rows) |
| m["mode_adherence"]={"passed":mode_passed,"total":len(mode_rows)} |
| m["quality_gate_passed"] &= mode_passed==len(mode_rows) |
| m["thinking_output_characters"]={name:sum(len(r["output"]) for r in rows if r["mode"]=="thinking")/sum(r["mode"]=="thinking" for r in rows) for name,rows in [("base",outputs["base_outputs.json"]),("astheral",outputs["astheral_merged_outputs.json"])]} |
| changed["evaluation.json"]=json.dumps(m,indent=2) |
| changed["evaluate.py"]=Path(__file__).with_name("evaluate.py").read_text() |
| |
| train=read("train.py") |
| train=train.replace('s["number"]["passed"]>=14','s["number"]["passed"]>=max(14,summary(baseline)["number"]["passed"])') |
| changed["train.py"]=train |
| settings=read("settings.json") |
| fingerprint=hashlib.sha256(("NousResearch/DeepHermes-3-Llama-3-8B-Preview"+"53d902c7dcfa4c749a3b455149064f82af2e2549"+changed["evaluate.py"]+settings).encode()).hexdigest() |
| changed["v2-baseline.json"]=json.dumps({"fingerprint":fingerprint,"outputs":outputs["base_outputs.json"],"parser_corrected_without_regenerating":True}) |
| manifest=json.loads(read("artifact_manifest.json")) |
| for name,body in changed.items():manifest[name]={"bytes":len(body.encode()),"sha256":hashlib.sha256(body.encode()).hexdigest()} |
| changed["artifact_manifest.json"]=json.dumps(manifest,indent=2) |
| api.create_commit(repo_id=repo,operations=[CommitOperationAdd(path_in_repo=n,path_or_fileobj=v.encode()) for n,v in changed.items()],commit_message="Correct numerical answer extraction consistently for base and Astheral evaluations") |
| print("CORRECTED_EVALUATION",json.dumps({"base":b,"astheral":s,"quality_gate_passed":m["quality_gate_passed"]}),flush=True) |
|
|
