#!/usr/bin/env python
"""Build the held-out eval set from OFFICIAL TEST splits only + emit the leakage manifest.
Firewall (handover §5): naamapadam TEST, IndicSentiment TEST, AVeriTeC DEV,
RAGTruth TEST, MMLU/ARC TEST. None of these appear in any training bucket."""
import os,sys,json,glob,zipfile,random,hashlib
sys.path.insert(0,"/workspace/aditi"); import contract as C
import pandas as pd
GH="/workspace/aditi/ghdata"; OUT="/workspace/aditi/evaldata"; os.makedirs(OUT,exist_ok=True)
random.seed(1234)
PER=200; PERLANG=40
def w(name,rows):
    with open(os.path.join(OUT,name),"w",encoding="utf-8") as f:
        for r in rows: f.write(json.dumps(r,ensure_ascii=False)+"\n")
    print(f"  {name:24s} {len(rows)}")
    return rows
man=[]
def hold(t): man.append(C.content_hash(t))

# --- NER: naamapadam TEST split, 11 languages
rows=[]
for z in sorted(glob.glob(f"{GH}/naamapadam/data/*_IndicNER_v1.0.zip")):
    lg=os.path.basename(z).split("_")[0]; zf=zipfile.ZipFile(z)
    te=[n for n in zf.namelist() if "test" in n.lower()]
    if not te: continue
    got=0
    with zf.open(te[0]) as fh:
        for line in fh:
            if got>=PERLANG: break
            try: r=json.loads(line)
            except Exception: continue
            ents=C.bio_entities(r["words"], r.get("ner") or [])
            txt=" ".join(r["words"])
            if len(txt)<25: continue
            rows.append({"task":"ner","lang":lg,"text":txt,"gold":{"entities":ents}}); hold(txt); got+=1
w("ner_test.jsonl",rows)
# --- sentiment: IndicSentiment TEST
rows=[]
for f in sorted(glob.glob(f"{GH}/IndicSentiment/data/test/*.json")):
    lg=os.path.basename(f).split(".")[0]; got=0
    try: rs=[json.loads(l) for l in open(f,encoding="utf-8") if l.strip()]
    except Exception: continue
    for r in rs:
        if got>=PERLANG: break
        t=str(r.get("INDIC REVIEW") or r.get("ENGLISH REVIEW") or "")
        g=C.canon("sentiment",str(r.get("LABEL","")).lower(),strict=False)
        if len(t)<20 or not g: continue
        rows.append({"task":"sentiment","lang":lg,"text":t,"gold":{"sentiment":g}}); hold(t); got+=1
w("sentiment_test.jsonl",rows)
# --- claim: AVeriTeC DEV (official test is blind)
rows=[]
d=pd.read_parquet(f"{GH}/averitec/data/dev-00000-of-00001.parquet")
for _,r in d.iterrows():
    if len(rows)>=PER: break
    v=C.AVERITEC_MAP.get(str(r["label"]))
    if not v: continue
    try: ev=" ".join(f"{q.get('question','')} {q.get('answers','')}" for q in r["questions"])[:1500]
    except Exception: continue
    if len(ev.strip())<20: continue
    rows.append({"task":"claim","claim":str(r["claim"])[:500],"evidence":ev,"gold":{"verdict":v}})
    hold(str(r["claim"]))
w("claim_dev.jsonl",rows)
# --- halluc_detect + faithfulness: RAGTruth TEST, split by context hash (disjoint halves)
d=pd.read_parquet(f"{GH}/RAGTruth-processed/data/test-00000-of-00001.parquet")
def hl(x):
    if isinstance(x,str):
        try: return json.loads(x)
        except Exception: return []
    return list(x or [])
d=d.copy(); d["_hl"]=d["hallucination_labels"].apply(hl)
ha,hb=[],[]
for _,r in d.iterrows():
    ctx=str(r["context"]); half=int(hashlib.sha256(ctx.encode()).hexdigest(),16)%2
    (ha if half==0 else hb).append(r)
rows=[]
for r in ha[:PER]:
    rows.append({"task":"halluc_detect","source":str(r["context"])[:1500],
                 "response":str(r["output"])[:900],"gold":{"grounded":len(r["_hl"])==0}})
    hold(str(r["context"]))
w("halluc_test_a.jsonl",rows)
rows=[]
for r in hb[:PER]:
    if len(r["_hl"])!=0: continue
    rows.append({"task":"faithfulness","source":str(r["context"])[:1500],
                 "question":str(r["query"])[:400],"gold":{"answer":str(r["output"])[:1200]}})
    hold(str(r["context"]))
w("faithfulness_test_b.jsonl",rows)
# --- general MCQ: frozen 1000, 250 each from mmlu / ai2_arc / arc_hi / mmlu-indic-bn
LET="ABCDEFGHIJ"; rows=[]
def add_mmlu(pat,src,n):
    got=0
    for f in sorted(glob.glob(pat)):
        try: dd=pd.read_parquet(f)
        except Exception: continue
        for _,r in dd.iterrows():
            if got>=n: return
            try: ch=list(r["choices"]); a=int(r["answer"])
            except Exception: continue
            q=str(r["question"]); cs="\n".join(f"{LET[i]}. {c}" for i,c in enumerate(ch))
            rows.append({"task":"general","src":src,"question":q,"choices":cs,
                         "gold":{"answer":LET[a]}}); hold(q); got+=1
add_mmlu(f"{GH}/mmlu/*/test-*.parquet","mmlu",250)
add_mmlu(f"{GH}/mmlu-indic/**/*bn*.parquet","mmlu-indic-bn",250)
for sub,src in (("ARC-Challenge","ai2_arc"),):
    for f in glob.glob(f"{GH}/ai2_arc/{sub}/test-*.parquet"):
        dd=pd.read_parquet(f); got=0
        for _,r in dd.iterrows():
            if got>=250: break
            ch=r["choices"]; texts=list(ch["text"]); labs=list(ch["label"])
            k=str(r["answerKey"])
            if k not in labs or labs.index(k)>=10: continue
            cs="\n".join(f"{LET[i]}. {t}" for i,t in enumerate(texts))
            rows.append({"task":"general","src":src,"question":str(r["question"]),
                         "choices":cs,"gold":{"answer":LET[labs.index(k)]}}); hold(str(r["question"])); got+=1
w("general_mcq.jsonl",rows)
# --- schema_lock stress: COMI-LINGUA LID_test
rows=[]
f=f"{GH}/COMI-LINGUA/LID_test.csv"
if os.path.exists(f):
    dd=pd.read_csv(f,encoding="utf-8-sig",on_bad_lines="skip")
    dd.columns=[c.strip().lower() for c in dd.columns]
    tc=next((c for c in ("text","sentence","tokens") if c in dd.columns),dd.columns[0])
    for _,r in dd.head(PER).iterrows():
        t=str(r[tc])[:1500]
        if len(t)<30: continue
        rows.append({"task":"schema_lock","text":t,"gold":None}); hold(t)
w("schema_lock_test.jsonl",rows)
uniq=sorted(set(man))
with open("/workspace/aditi/eval_heldout_manifest.jsonl","w") as f:
    f.write(json.dumps({"_meta":{"n_items":len(man),"n_unique":len(uniq)}})+"\n")
    for h in uniq: f.write(json.dumps({"sha256":h})+"\n")
print(f"\nheldout manifest: {len(man)} items -> {len(uniq)} unique sha256")