| |
| """Build the held-out eval set from OFFICIAL TEST splits only + emit the leakage manifest. |
| Firewall (handover §5): naamapadam TEST, IndicSentiment TEST, AVeriTeC DEV, |
| RAGTruth TEST, MMLU/ARC TEST. None of these appear in any training bucket.""" |
| import os,sys,json,glob,zipfile,random,hashlib |
| sys.path.insert(0,"/workspace/aditi"); import contract as C |
| import pandas as pd |
| GH="/workspace/aditi/ghdata"; OUT="/workspace/aditi/evaldata"; os.makedirs(OUT,exist_ok=True) |
| random.seed(1234) |
| PER=200; PERLANG=40 |
| def w(name,rows): |
| with open(os.path.join(OUT,name),"w",encoding="utf-8") as f: |
| for r in rows: f.write(json.dumps(r,ensure_ascii=False)+"\n") |
| print(f" {name:24s} {len(rows)}") |
| return rows |
| man=[] |
| def hold(t): man.append(C.content_hash(t)) |
|
|
| |
| rows=[] |
| for z in sorted(glob.glob(f"{GH}/naamapadam/data/*_IndicNER_v1.0.zip")): |
| lg=os.path.basename(z).split("_")[0]; zf=zipfile.ZipFile(z) |
| te=[n for n in zf.namelist() if "test" in n.lower()] |
| if not te: continue |
| got=0 |
| with zf.open(te[0]) as fh: |
| for line in fh: |
| if got>=PERLANG: break |
| try: r=json.loads(line) |
| except Exception: continue |
| ents=C.bio_entities(r["words"], r.get("ner") or []) |
| txt=" ".join(r["words"]) |
| if len(txt)<25: continue |
| rows.append({"task":"ner","lang":lg,"text":txt,"gold":{"entities":ents}}); hold(txt); got+=1 |
| w("ner_test.jsonl",rows) |
| |
| rows=[] |
| for f in sorted(glob.glob(f"{GH}/IndicSentiment/data/test/*.json")): |
| lg=os.path.basename(f).split(".")[0]; got=0 |
| try: rs=[json.loads(l) for l in open(f,encoding="utf-8") if l.strip()] |
| except Exception: continue |
| for r in rs: |
| if got>=PERLANG: break |
| t=str(r.get("INDIC REVIEW") or r.get("ENGLISH REVIEW") or "") |
| g=C.canon("sentiment",str(r.get("LABEL","")).lower(),strict=False) |
| if len(t)<20 or not g: continue |
| rows.append({"task":"sentiment","lang":lg,"text":t,"gold":{"sentiment":g}}); hold(t); got+=1 |
| w("sentiment_test.jsonl",rows) |
| |
| rows=[] |
| d=pd.read_parquet(f"{GH}/averitec/data/dev-00000-of-00001.parquet") |
| for _,r in d.iterrows(): |
| if len(rows)>=PER: break |
| v=C.AVERITEC_MAP.get(str(r["label"])) |
| if not v: continue |
| try: ev=" ".join(f"{q.get('question','')} {q.get('answers','')}" for q in r["questions"])[:1500] |
| except Exception: continue |
| if len(ev.strip())<20: continue |
| rows.append({"task":"claim","claim":str(r["claim"])[:500],"evidence":ev,"gold":{"verdict":v}}) |
| hold(str(r["claim"])) |
| w("claim_dev.jsonl",rows) |
| |
| d=pd.read_parquet(f"{GH}/RAGTruth-processed/data/test-00000-of-00001.parquet") |
| def hl(x): |
| if isinstance(x,str): |
| try: return json.loads(x) |
| except Exception: return [] |
| return list(x or []) |
| d=d.copy(); d["_hl"]=d["hallucination_labels"].apply(hl) |
| ha,hb=[],[] |
| for _,r in d.iterrows(): |
| ctx=str(r["context"]); half=int(hashlib.sha256(ctx.encode()).hexdigest(),16)%2 |
| (ha if half==0 else hb).append(r) |
| rows=[] |
| for r in ha[:PER]: |
| rows.append({"task":"halluc_detect","source":str(r["context"])[:1500], |
| "response":str(r["output"])[:900],"gold":{"grounded":len(r["_hl"])==0}}) |
| hold(str(r["context"])) |
| w("halluc_test_a.jsonl",rows) |
| rows=[] |
| for r in hb[:PER]: |
| if len(r["_hl"])!=0: continue |
| rows.append({"task":"faithfulness","source":str(r["context"])[:1500], |
| "question":str(r["query"])[:400],"gold":{"answer":str(r["output"])[:1200]}}) |
| hold(str(r["context"])) |
| w("faithfulness_test_b.jsonl",rows) |
| |
| LET="ABCDEFGHIJ"; rows=[] |
| def add_mmlu(pat,src,n): |
| got=0 |
| for f in sorted(glob.glob(pat)): |
| try: dd=pd.read_parquet(f) |
| except Exception: continue |
| for _,r in dd.iterrows(): |
| if got>=n: return |
| try: ch=list(r["choices"]); a=int(r["answer"]) |
| except Exception: continue |
| q=str(r["question"]); cs="\n".join(f"{LET[i]}. {c}" for i,c in enumerate(ch)) |
| rows.append({"task":"general","src":src,"question":q,"choices":cs, |
| "gold":{"answer":LET[a]}}); hold(q); got+=1 |
| add_mmlu(f"{GH}/mmlu/*/test-*.parquet","mmlu",250) |
| add_mmlu(f"{GH}/mmlu-indic/**/*bn*.parquet","mmlu-indic-bn",250) |
| for sub,src in (("ARC-Challenge","ai2_arc"),): |
| for f in glob.glob(f"{GH}/ai2_arc/{sub}/test-*.parquet"): |
| dd=pd.read_parquet(f); got=0 |
| for _,r in dd.iterrows(): |
| if got>=250: break |
| ch=r["choices"]; texts=list(ch["text"]); labs=list(ch["label"]) |
| k=str(r["answerKey"]) |
| if k not in labs or labs.index(k)>=10: continue |
| cs="\n".join(f"{LET[i]}. {t}" for i,t in enumerate(texts)) |
| rows.append({"task":"general","src":src,"question":str(r["question"]), |
| "choices":cs,"gold":{"answer":LET[labs.index(k)]}}); hold(str(r["question"])); got+=1 |
| w("general_mcq.jsonl",rows) |
| |
| rows=[] |
| f=f"{GH}/COMI-LINGUA/LID_test.csv" |
| if os.path.exists(f): |
| dd=pd.read_csv(f,encoding="utf-8-sig",on_bad_lines="skip") |
| dd.columns=[c.strip().lower() for c in dd.columns] |
| tc=next((c for c in ("text","sentence","tokens") if c in dd.columns),dd.columns[0]) |
| for _,r in dd.head(PER).iterrows(): |
| t=str(r[tc])[:1500] |
| if len(t)<30: continue |
| rows.append({"task":"schema_lock","text":t,"gold":None}); hold(t) |
| w("schema_lock_test.jsonl",rows) |
| uniq=sorted(set(man)) |
| with open("/workspace/aditi/eval_heldout_manifest.jsonl","w") as f: |
| f.write(json.dumps({"_meta":{"n_items":len(man),"n_unique":len(uniq)}})+"\n") |
| for h in uniq: f.write(json.dumps({"sha256":h})+"\n") |
| print(f"\nheldout manifest: {len(man)} items -> {len(uniq)} unique sha256") |
|
|
