Ai-detector-websterfly / scripts /train_fusion.py
Websterfly's picture
Deploy Websterfly educational AI detection workbench
97799b9 verified
Raw History Blame Contribute Delete
3.43 kB
"""Train a regularized fusion candidate on cached paired document scores.
No paper content, paper score, or commercial verdict is used for training.
"""
import argparse,json,math,hashlib
from pathlib import Path
import numpy as np
from sklearn.linear_model import LogisticRegression
from sklearn.model_selection import GroupShuffleSplit
from sklearn.metrics import roc_auc_score,confusion_matrix
from scipy.stats import binomtest
KEYS=['fakespot','earlybird','vanguard']
def logit(s):
s=min(max(float(s),1e-6),1-1e-6)
return math.log(s/(1-s))
def main():
p=argparse.ArgumentParser();p.add_argument('--scores',type=Path,default=Path('work/upgrade'));p.add_argument('--output',type=Path,default=Path('artifacts'));a=p.parse_args()
sources={k:[json.loads(l) for l in (a.scores/(k+'-external.jsonl')).read_text().splitlines()] for k in KEYS}
base=sources[KEYS[0]]
identities=[(r['group'],r['label']) for r in base]
if len(set(identities))!=len(identities):raise ValueError('Duplicate document identities')
for records in sources.values():
if [(r['group'],r['label']) for r in records]!=identities:raise ValueError('Model score rows do not align')
x=np.array([[logit(sources[k][i]['ai_score']) for k in KEYS] for i in range(len(base))]);y=np.array([r['label']=='ai' for r in base],dtype=int);groups=[r['group'] for r in base]
train,test=next(GroupShuffleSplit(n_splits=1,test_size=.3334,random_state=731).split(x,y,groups))
model=LogisticRegression(C=1,max_iter=1000).fit(x[train],y[train])
def metrics(scores):
tn,fp,fn,tp=confusion_matrix(y[test],np.array(scores)>=.5,labels=[0,1]).ravel()
ci=binomtest(int(fp),int(fp+tn)).proportion_ci(confidence_level=.95,method='exact')
return {'false_positive_rate_95pct_interval':[float(ci.low),float(ci.high)],'auroc':float(roc_auc_score(y[test],scores)),'false_positive_rate':float(fp/(fp+tn)),'detection_rate':float(tp/(tp+fn)),'human_documents':int(fp+tn),'ai_documents':int(tp+fn),'threshold':.5}
report={'status':'pilot_only_not_release_validated','train_documents':len(train),'test_documents':len(test),'train_groups':len(set(groups[i] for i in train)),'test_groups':len(set(groups[i] for i in test)),
'split_seed':731,'paper_excluded':True,'ensemble':metrics(model.predict_proba(x[test])[:,1]),'individual':{k:metrics([sources[k][i]['ai_score'] for i in test]) for k in KEYS},
'limitations':['Control inputs use fixed first-300-word excerpts; long-paper aggregation is not validated by this pilot.','Tiny convenience sample: no low-FPR or market-superiority claim is supported.','Checkpoint pretraining overlap is unknown; this is not an independent benchmark.','Domain/generator subgroup sizes are too small for meaningful estimates.','Test outcomes are reported without tuning model selection or threshold.']}
from app.detectors import SPECS
artifact={'status':report['status'],'features':[{ 'key':k,'model_id':SPECS[k]['model_id'],'revision':SPECS[k]['revision']} for k in KEYS],'transform':'clipped_logit','coefficients':model.coef_[0].tolist(),'intercept':float(model.intercept_[0]),'calibrated':False,'training_manifest':json.loads((a.scores/'dataset-manifest.json').read_text()),'validation':report}
a.output.mkdir(exist_ok=True);(a.output/'pilot-fusion.json').write_text(json.dumps(artifact,indent=2));(a.output/'pilot-validation.json').write_text(json.dumps(report,indent=2))
print(json.dumps(report,indent=2))
if __name__=='__main__':main()