Blame for reproduce/neural_three_benchmarks.py

"""Neural-only evaluation for three user-uploaded RAG/evidence datasets.
All ranking uses original MiniLM cosine / pretrained MS-MARCO cross encoder.
Graph is a fixed previously specified directed-link/title selection policy; gold is
never accessed by scoring/selection. QASC has TWO explicitly separate protocols.
"""
import os
os.environ['OPENBLAS_NUM_THREADS']='1'; os.environ['MKL_NUM_THREADS']='1'; os.environ['OMP_NUM_THREADS']='4'
import json, time, struct, collections, sys
from pathlib import Path
import numpy as np
import torch
from safetensors.torch import load_file
from minilm_cpu_pipeline import MiniLM, WordPiece, embed
from neural_crossencoder_only import load_crossencoder,PairTokenizer,score_pairs
from neural_evidence_graph_core import (hover,graph_arrays,graph_select,seeded_sample,doc,ROOT)
from parquet_lite import read_col,reconstruct
from parquet_probe import parse

K=(2,3,5,8)

def get_dense():
 m=MiniLM();sd=load_file(str(ROOT/'original_minilm/model.safetensors'))
 for key in ('embeddings.position_ids','pooler.dense.bias','pooler.dense.weight'):sd.pop(key)
 m.load_state_dict(sd,strict=True);m.eval();torch.set_num_threads(4)
 return m,WordPiece(str(ROOT/'vocab.txt'),length=128)

def embed_texts(texts,m,tok,batch=32,label=''):
 t=time.perf_counter();out=embed(texts,m,tok,batch_size=batch)
 print('DENSE_EMBED',label,len(texts),'seconds',round(time.perf_counter()-t,3),flush=True)
 return out,time.perf_counter()-t

def get_qasc():
 p=ROOT/'train-00000-of-00001.parquet';f=open(p,'rb');f.seek(-8,2)
 size=struct.unpack('<I',f.read(4))[0];f.seek(-8-size,2);meta=parse(f.read(size));rows=[]
 for rg in meta[4]:
  cols={}
  for c in rg[1]:
   key=tuple(c[3][3])
   if key not in [('id',),('question',),('fact1',),('fact2',)]:continue
   cols[key]=reconstruct(*read_col(f,c,{key:(1,0)}),1,0)
  rows.extend([{'id':cols[('id',)][i], 'question':cols[('question',)][i], 'facts':[cols[('fact1',)][i],cols[('fact2',)][i]]} for i in range(rg[3])])
 rows=[r for r in rows if all(x and str(x).strip() for x in r['facts']) and r['facts'][0]!=r['facts'][1]]
 facts=list(dict.fromkeys(z.strip() for r in rows for z in r['facts']))
 return rows,facts

def prepare_multihop(n,m,tok):
 t=time.perf_counter();corpus=json.load(open(ROOT/'corpus.json'));allq=json.load(open(ROOT/'MultiHopRAG.json'))
 eligible=[r for r in allq if len({e['url'] for e in r['evidence_list']})>=2]
 questions=seeded_sample(eligible,n)
 texts=[r['title']+'. '+r['body'][:1800] for r in corpus]
 de,tidx=embed_texts(texts,m,tok,label='MultiHop corpus')
 qe,tq=embed_texts([r['query'] for r in questions],m,tok,label='MultiHop queries')
 sim=qe@de.T;inds=np.argsort(-sim,axis=1,kind='stable')[:,:20]
 rows=[];sc=[]
 for i,r in enumerate(questions):
  docs=[doc(corpus[j]['title'],corpus[j]['body'],corpus[j]['url']) for j in inds[i]]
  rows.append({'qid':f'multihop:{i}', 'query':r['query'],'docs':docs,'gold':{e['url'] for e in r['evidence_list']},'pool':'Top-20 neural MiniLM retrieval from full 609-article corpus (no gold insertion)'})
  sc.append(sim[i,inds[i]])
 return rows,sc,{'index_seconds':tidx,'query_embedding_seconds':tq,'total_prep_seconds':time.perf_counter()-t,'index_size':len(corpus),'candidate_pool_size':20}

def prepare_hover(n,m,tok):
 t=time.perf_counter();rows=hover(n)
 # Exact 10 candidate passages; labels sourced from corresponding HoVer claims.
 alltexts=[r['query'] for r in rows]+[d['title']+'. '+d['body'] for r in rows for d in r['docs']]
 em,sec=embed_texts(alltexts,m,tok,label='HoVer queries+passages')
 Q=em[:len(rows)]; Ds=np.split(em[len(rows):],np.cumsum([len(r['docs']) for r in rows])[:-1]);sc=[Ds[i]@Q[i] for i in range(len(rows))]
 return rows,sc,{'embedding_seconds':sec,'prep_seconds':time.perf_counter()-t,'candidate_pool_size':10,'eligibility':'HoVer dev claims with HotpotQA dev passages available and all annotated support titles present; conditional subset'}

def prepare_qasc(n,m,tok):
 t=time.perf_counter();allrows,facts=get_qasc();qs=seeded_sample(allrows,n)
 fe,tidx=embed_texts(facts,m,tok,label='QASC all distinct supporting fact sentences')
 qe,tq=embed_texts([r['question'] for r in qs],m,tok,label='QASC query')
 sim=qe@fe.T;inds=np.argsort(-sim,axis=1,kind='stable')[:,:60]
 rows_native=[];scores_native=[];rows_ctrl=[];scores_ctrl=[]
 for i,r in enumerate(qs):
  gold={str(x).strip() for x in r['facts']}
  docs=[doc('',facts[int(j)],facts[int(j)]) for j in inds[i,:20]]
  rows_native.append({'qid':r['id'],'query':r['question'],'docs':docs,'gold':gold,'pool':'Top-20 MiniLM from all unique QASC supporting fact sentences, without gold insertion'})
  scores_native.append(sim[i,inds[i,:20]])
  neg=[int(j) for j in inds[i] if facts[int(j)] not in gold][:18]
  if len(neg)!=18:continue
  chosen=list(gold)+[facts[j] for j in neg]
  perm=np.random.default_rng(i+301).permutation(len(chosen))
  chosen=[chosen[int(j)] for j in perm]
  look={f:j for j,f in enumerate(facts)}
  rows_ctrl.append({'qid':r['id'],'query':r['question'],'docs':[doc('',z,z) for z in chosen],'gold':gold,'pool':'CONTROLLED: 2 gold facts deliberately inserted + 18 MiniLM hard negatives (oracle=100%)'})
  scores_ctrl.append(np.array([sim[i,look[z]] for z in chosen],np.float32))
 return (rows_native,scores_native),(rows_ctrl,scores_ctrl),{'index_seconds':tidx,'query_embedding_seconds':tq,'total_prep_seconds':time.perf_counter()-t,'fact_corpus_size':len(facts),'eligible_questions':len(allrows),'candidate_pool_size':20}

def assess(name,rows,dense_scores,cross_scores,loadtime,extra):
 tgraph=time.perf_counter();details=[]
 counts={method:{str(k):[] for k in K} for method in ('dense_cosine','crossencoder','dense_graph','crossencoder_graph')}
 oracle={str(k):[] for k in K};av=[];links=[];sizecnt=collections.Counter()
 for i,r in enumerate(rows):
  gold=r['gold'];docs=r['docs'];ss=np.asarray(dense_scores[i],dtype=np.float32);cc=np.asarray(cross_scores[i],dtype=np.float32)
  assert len(ss)==len(cc)==len(docs)
  link,match=graph_arrays(r);links.append(bool(link.any()))
  avail=len(gold&{d['id'] for d in docs});av.append(avail==len(gold));sizecnt[len(gold)]+=1
  orders={
   'dense_cosine':np.argsort(-ss,kind='stable').tolist(),
   'crossencoder':np.argsort(-cc,kind='stable').tolist(),
   'dense_graph':graph_select(ss,link,match,max(K)),
   'crossencoder_graph':graph_select(cc,link,match,max(K)),
  }
  record={'id':r['qid'],'gold_count':len(gold),'pool_gold_count':avail,'orders':orders,'hits':{}}
  for k in K:
   oracle[str(k)].append(int(avail==len(gold) and len(gold)<=k))
   for method,order in orders.items():
    hit=int(gold.issubset({docs[j]['id'] for j in order[:k]}))
    counts[method][str(k)].append(hit)
    record['hits'].setdefault(method,{})[str(k)]=hit
  details.append(record)
 graphseconds=time.perf_counter()-tgraph
 delta=np.array(counts['crossencoder_graph']['3'],int)-np.array(counts['crossencoder']['3'],int)
 if len(delta):
  gen=np.random.default_rng(31);ci=np.percentile(delta[gen.integers(0,len(delta),(2000,len(delta)))].mean(axis=1)*100,[2.5,97.5])
 else:ci=[float('nan')]*2
 res={'dataset':name,'questions':len(rows),'candidate_count_total':sum(len(r['docs']) for r in rows),'candidate_pool':rows[0]['pool'] if rows else '',
   'gold_support_count_distribution':dict(sizecnt),'gold_available_fraction':round(float(np.mean(av))*100,2),'gold_available_count':sum(av), 'graph_link_fraction':round(float(np.mean(links))*100,2),
   'oracle_complete_coverage_pct':{k:round(float(np.mean(v))*100,2) for k,v in oracle.items()},
   'complete_coverage_pct':{met:{k:round(float(np.mean(val))*100,2) for k,val in d.items()} for met,d in counts.items()},
   'crossencoder_graph_minus_crossencoder_at3_pp':round(float(delta.mean()*100),2),'ci95_pp':[round(float(z),2) for z in ci],
   'paired_wins_losses_ties_at3':{'wins':int((delta>0).sum()),'losses':int((delta<0).sum()),'ties':int((delta==0).sum())},
   'timing_seconds':{**extra,**loadtime,'graph_selection_total_s':round(graphseconds,3),'graph_selection_ms_per_question':round(graphseconds*1000/max(1,len(rows)),2)},
   'models':{'dense':'user uploaded all-MiniLM-L6-v2', 'crossencoder':'user uploaded cross-encoder/ms-marco-MiniLM-L6-v2'},
   'protocol':'MiniLM embeddings at max 128 WordPiece for dense. Cross-encoder truncates paired query+passage at 256 WordPiece. Both score exact same candidate pools. Fixed evidence graph .4 minmax relevance + .5 title in already-selected passage + .2 query-title score. No TF-IDF anywhere in candidate construction or scoring. Gold used solely for pool audit and evaluation.'}
 (ROOT/f'{name}_neural_only_results.json').write_text(json.dumps(res,indent=2))
 (ROOT/f'{name}_neural_only_per_question.json').write_text(json.dumps(details))
 print('RESULT',name,json.dumps(res),flush=True)
 return res

def evaluate(name,rows,denses,embtimes,ce,tok):
 pairs=[(r['query'],(d['title']+'. ' if d['title'] else '')+d['body']) for r in rows for d in r['docs']]
 print('CROSS_START',name,'n',len(rows),'pairs',len(pairs),flush=True)
 t=time.perf_counter();cross,perf=score_pairs(ce,tok,pairs,batch=20)
 tsec=time.perf_counter()-t
 lengths=np.cumsum([len(r['docs']) for r in rows])[:-1]
 scores=np.split(cross,lengths)
 report=assess(name,rows,denses,scores,{'crossencoder_scoring_seconds':round(tsec,3),'crossencoder_scoring_ms_per_question':round(tsec*1000/len(rows),2),'crossencoder_forward_seconds':round(perf['total_forward_s'],3)},embtimes)
 return report

if __name__=='__main__':
 n=int(sys.argv[1]) if len(sys.argv)>1 else 300
 ds=sys.argv[2:] or ['hover','multihop','qasc']
 dense,tok=get_dense();ce=load_crossencoder();ctok=PairTokenizer(length=256)
 for task in ds:
  if task=='hover':
   rows,scores,times=prepare_hover(n,dense,tok)
   evaluate('hover',rows,scores,times,ce,ctok)
  elif task=='multihop':
   rows,scores,times=prepare_multihop(n,dense,tok)
   evaluate('multihop',rows,scores,times,ce,ctok)
  elif task=='qasc':
   (rows,scores),(ctrl,cscores),times=prepare_qasc(n,dense,tok)
   evaluate('qasc_neural_retrieval',rows,scores,times,ce,ctok)
   evaluate('qasc_controlled_pool',ctrl,cscores,{'cached_fact_embeddings_used':True},ce,ctok)
  else:raise ValueError(task)