| """Small hand-constructed abstention sanity check, not a published benchmark.""" |
| import argparse,json,time |
| from evaluate_model import ROOT,Adapter,dump,counts,metric |
| from investigate_models import predict_documents,flat_predictions |
| import numpy as np |
|
|
| def main(): |
| p=argparse.ArgumentParser();p.add_argument('--model',required=True);args=p.parse_args() |
| entry=next(e for e in json.loads((ROOT/'run_models.json').read_text()) if e['id']==args.model) |
| inv=json.loads((ROOT/'audit/labels.json').read_text()) |
| schema={k:inv[k]['definitions'][0]['definition'] for k in ['person','company','city','court','protein','disease']} |
| negatives=['There are three boxes on the shelf.','The report contains two tables.','The door is open.', |
| 'The answer is unknown.','Nothing was added.','The file is empty.','The numbers have changed.', |
| 'The lights are on.','The window is closed.','The basket is full.','The path is narrow.','The signal is weak.'] |
| docs=[{'id':f'negative:{i}','text':t,'entities':[]} for i,t in enumerate(negatives)] |
| for i,(name,company,city) in enumerate([('Maria Lopez','Acme Corporation','Paris'),('James Wilson','Microsoft','London'), |
| ('Aisha Khan','Google','Tokyo'),('Daniel Lee','Apple','Berlin'),('Sofia Rossi','IBM','Rome'),('Emma Brown','Amazon','Madrid')]): |
| text=f'{name} works for {company} in {city}.';entities=[] |
| for mention,label in [(name,'person'),(company,'company'),(city,'city')]: |
| s=text.index(mention);entities.append([s,s+len(mention),label]) |
| docs.append({'id':f'positive:{i}','text':text,'entities':entities}) |
| adapter=Adapter(entry);results=[] |
| for threshold in [.5,.65]: |
| pred,runtime=predict_documents(adapter,docs,schema,threshold=threshold,max_len=512 if 'specialised' in entry['id'] else 3072) |
| for policy,values in [('native',pred),('global_flat',flat_predictions(pred))]: |
| rows=[];total=np.zeros(3,dtype=int) |
| for doc,items in zip(docs,values): |
| clean=[[s,e,l] for s,e,l,c in items];c=counts(doc['entities'],clean);total+=c |
| rows.append({'id':doc['id'],'text':doc['text'],'gold':doc['entities'],'predictions':items,'counts':c}) |
| results.append({'threshold':threshold,'policy':policy,'metrics':metric(total), |
| 'all_negative_documents_with_predictions':sum(bool(v) for v in values[:len(negatives)]), |
| 'all_negative_documents':len(negatives),'rows':rows}) |
| dump(ROOT/'investigation'/entry['id'].replace('/','--')/'negative-query-sanity.json',{ |
| 'scope':'18 hand-constructed diagnostic examples; no population-level generalization claim', |
| 'schema':schema,'results':results}) |
|
|
| if __name__=='__main__':main() |
|
|
