File size: 6,034 Bytes
6c8db72 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 | #!/usr/bin/env python
"""Build the held-out eval set from OFFICIAL TEST splits only + emit the leakage manifest.
Firewall (handover §5): naamapadam TEST, IndicSentiment TEST, AVeriTeC DEV,
RAGTruth TEST, MMLU/ARC TEST. None of these appear in any training bucket."""
import os,sys,json,glob,zipfile,random,hashlib
sys.path.insert(0,"/workspace/aditi"); import contract as C
import pandas as pd
GH="/workspace/aditi/ghdata"; OUT="/workspace/aditi/evaldata"; os.makedirs(OUT,exist_ok=True)
random.seed(1234)
PER=200; PERLANG=40
def w(name,rows):
with open(os.path.join(OUT,name),"w",encoding="utf-8") as f:
for r in rows: f.write(json.dumps(r,ensure_ascii=False)+"\n")
print(f" {name:24s} {len(rows)}")
return rows
man=[]
def hold(t): man.append(C.content_hash(t))
# --- NER: naamapadam TEST split, 11 languages
rows=[]
for z in sorted(glob.glob(f"{GH}/naamapadam/data/*_IndicNER_v1.0.zip")):
lg=os.path.basename(z).split("_")[0]; zf=zipfile.ZipFile(z)
te=[n for n in zf.namelist() if "test" in n.lower()]
if not te: continue
got=0
with zf.open(te[0]) as fh:
for line in fh:
if got>=PERLANG: break
try: r=json.loads(line)
except Exception: continue
ents=C.bio_entities(r["words"], r.get("ner") or [])
txt=" ".join(r["words"])
if len(txt)<25: continue
rows.append({"task":"ner","lang":lg,"text":txt,"gold":{"entities":ents}}); hold(txt); got+=1
w("ner_test.jsonl",rows)
# --- sentiment: IndicSentiment TEST
rows=[]
for f in sorted(glob.glob(f"{GH}/IndicSentiment/data/test/*.json")):
lg=os.path.basename(f).split(".")[0]; got=0
try: rs=[json.loads(l) for l in open(f,encoding="utf-8") if l.strip()]
except Exception: continue
for r in rs:
if got>=PERLANG: break
t=str(r.get("INDIC REVIEW") or r.get("ENGLISH REVIEW") or "")
g=C.canon("sentiment",str(r.get("LABEL","")).lower(),strict=False)
if len(t)<20 or not g: continue
rows.append({"task":"sentiment","lang":lg,"text":t,"gold":{"sentiment":g}}); hold(t); got+=1
w("sentiment_test.jsonl",rows)
# --- claim: AVeriTeC DEV (official test is blind)
rows=[]
d=pd.read_parquet(f"{GH}/averitec/data/dev-00000-of-00001.parquet")
for _,r in d.iterrows():
if len(rows)>=PER: break
v=C.AVERITEC_MAP.get(str(r["label"]))
if not v: continue
try: ev=" ".join(f"{q.get('question','')} {q.get('answers','')}" for q in r["questions"])[:1500]
except Exception: continue
if len(ev.strip())<20: continue
rows.append({"task":"claim","claim":str(r["claim"])[:500],"evidence":ev,"gold":{"verdict":v}})
hold(str(r["claim"]))
w("claim_dev.jsonl",rows)
# --- halluc_detect + faithfulness: RAGTruth TEST, split by context hash (disjoint halves)
d=pd.read_parquet(f"{GH}/RAGTruth-processed/data/test-00000-of-00001.parquet")
def hl(x):
if isinstance(x,str):
try: return json.loads(x)
except Exception: return []
return list(x or [])
d=d.copy(); d["_hl"]=d["hallucination_labels"].apply(hl)
ha,hb=[],[]
for _,r in d.iterrows():
ctx=str(r["context"]); half=int(hashlib.sha256(ctx.encode()).hexdigest(),16)%2
(ha if half==0 else hb).append(r)
rows=[]
for r in ha[:PER]:
rows.append({"task":"halluc_detect","source":str(r["context"])[:1500],
"response":str(r["output"])[:900],"gold":{"grounded":len(r["_hl"])==0}})
hold(str(r["context"]))
w("halluc_test_a.jsonl",rows)
rows=[]
for r in hb[:PER]:
if len(r["_hl"])!=0: continue
rows.append({"task":"faithfulness","source":str(r["context"])[:1500],
"question":str(r["query"])[:400],"gold":{"answer":str(r["output"])[:1200]}})
hold(str(r["context"]))
w("faithfulness_test_b.jsonl",rows)
# --- general MCQ: frozen 1000, 250 each from mmlu / ai2_arc / arc_hi / mmlu-indic-bn
LET="ABCDEFGHIJ"; rows=[]
def add_mmlu(pat,src,n):
got=0
for f in sorted(glob.glob(pat)):
try: dd=pd.read_parquet(f)
except Exception: continue
for _,r in dd.iterrows():
if got>=n: return
try: ch=list(r["choices"]); a=int(r["answer"])
except Exception: continue
q=str(r["question"]); cs="\n".join(f"{LET[i]}. {c}" for i,c in enumerate(ch))
rows.append({"task":"general","src":src,"question":q,"choices":cs,
"gold":{"answer":LET[a]}}); hold(q); got+=1
add_mmlu(f"{GH}/mmlu/*/test-*.parquet","mmlu",250)
add_mmlu(f"{GH}/mmlu-indic/**/*bn*.parquet","mmlu-indic-bn",250)
for sub,src in (("ARC-Challenge","ai2_arc"),):
for f in glob.glob(f"{GH}/ai2_arc/{sub}/test-*.parquet"):
dd=pd.read_parquet(f); got=0
for _,r in dd.iterrows():
if got>=250: break
ch=r["choices"]; texts=list(ch["text"]); labs=list(ch["label"])
k=str(r["answerKey"])
if k not in labs or labs.index(k)>=10: continue
cs="\n".join(f"{LET[i]}. {t}" for i,t in enumerate(texts))
rows.append({"task":"general","src":src,"question":str(r["question"]),
"choices":cs,"gold":{"answer":LET[labs.index(k)]}}); hold(str(r["question"])); got+=1
w("general_mcq.jsonl",rows)
# --- schema_lock stress: COMI-LINGUA LID_test
rows=[]
f=f"{GH}/COMI-LINGUA/LID_test.csv"
if os.path.exists(f):
dd=pd.read_csv(f,encoding="utf-8-sig",on_bad_lines="skip")
dd.columns=[c.strip().lower() for c in dd.columns]
tc=next((c for c in ("text","sentence","tokens") if c in dd.columns),dd.columns[0])
for _,r in dd.head(PER).iterrows():
t=str(r[tc])[:1500]
if len(t)<30: continue
rows.append({"task":"schema_lock","text":t,"gold":None}); hold(t)
w("schema_lock_test.jsonl",rows)
uniq=sorted(set(man))
with open("/workspace/aditi/eval_heldout_manifest.jsonl","w") as f:
f.write(json.dumps({"_meta":{"n_items":len(man),"n_unique":len(uniq)}})+"\n")
for h in uniq: f.write(json.dumps({"sha256":h})+"\n")
print(f"\nheldout manifest: {len(man)} items -> {len(uniq)} unique sha256")
|