Download scripts/train_fusion.py from Websterfly/Ai-detector-websterfly: direct link, hf CLI and curl.
- Browser
- Download file 3.43 kB
-
https://huggingface.co/spaces/Websterfly/Ai-detector-websterfly/resolve/main/scripts/train_fusion.py
- Command line
-
hf download hf://spaces/Websterfly/Ai-detector-websterfly/scripts/train_fusion.py
-
curl -L -o train_fusion.py https://huggingface.co/spaces/Websterfly/Ai-detector-websterfly/resolve/main/scripts/train_fusion.py
3.43 kB
| """Train a regularized fusion candidate on cached paired document scores. | |
| No paper content, paper score, or commercial verdict is used for training. | |
| """ | |
| import argparse,json,math,hashlib | |
| from pathlib import Path | |
| import numpy as np | |
| from sklearn.linear_model import LogisticRegression | |
| from sklearn.model_selection import GroupShuffleSplit | |
| from sklearn.metrics import roc_auc_score,confusion_matrix | |
| from scipy.stats import binomtest | |
| KEYS=['fakespot','earlybird','vanguard'] | |
| def logit(s): | |
| s=min(max(float(s),1e-6),1-1e-6) | |
| return math.log(s/(1-s)) | |
| def main(): | |
| p=argparse.ArgumentParser();p.add_argument('--scores',type=Path,default=Path('work/upgrade'));p.add_argument('--output',type=Path,default=Path('artifacts'));a=p.parse_args() | |
| sources={k:[json.loads(l) for l in (a.scores/(k+'-external.jsonl')).read_text().splitlines()] for k in KEYS} | |
| base=sources[KEYS[0]] | |
| identities=[(r['group'],r['label']) for r in base] | |
| if len(set(identities))!=len(identities):raise ValueError('Duplicate document identities') | |
| for records in sources.values(): | |
| if [(r['group'],r['label']) for r in records]!=identities:raise ValueError('Model score rows do not align') | |
| x=np.array([[logit(sources[k][i]['ai_score']) for k in KEYS] for i in range(len(base))]);y=np.array([r['label']=='ai' for r in base],dtype=int);groups=[r['group'] for r in base] | |
| train,test=next(GroupShuffleSplit(n_splits=1,test_size=.3334,random_state=731).split(x,y,groups)) | |
| model=LogisticRegression(C=1,max_iter=1000).fit(x[train],y[train]) | |
| def metrics(scores): | |
| tn,fp,fn,tp=confusion_matrix(y[test],np.array(scores)>=.5,labels=[0,1]).ravel() | |
| ci=binomtest(int(fp),int(fp+tn)).proportion_ci(confidence_level=.95,method='exact') | |
| return {'false_positive_rate_95pct_interval':[float(ci.low),float(ci.high)],'auroc':float(roc_auc_score(y[test],scores)),'false_positive_rate':float(fp/(fp+tn)),'detection_rate':float(tp/(tp+fn)),'human_documents':int(fp+tn),'ai_documents':int(tp+fn),'threshold':.5} | |
| report={'status':'pilot_only_not_release_validated','train_documents':len(train),'test_documents':len(test),'train_groups':len(set(groups[i] for i in train)),'test_groups':len(set(groups[i] for i in test)), | |
| 'split_seed':731,'paper_excluded':True,'ensemble':metrics(model.predict_proba(x[test])[:,1]),'individual':{k:metrics([sources[k][i]['ai_score'] for i in test]) for k in KEYS}, | |
| 'limitations':['Control inputs use fixed first-300-word excerpts; long-paper aggregation is not validated by this pilot.','Tiny convenience sample: no low-FPR or market-superiority claim is supported.','Checkpoint pretraining overlap is unknown; this is not an independent benchmark.','Domain/generator subgroup sizes are too small for meaningful estimates.','Test outcomes are reported without tuning model selection or threshold.']} | |
| from app.detectors import SPECS | |
| artifact={'status':report['status'],'features':[{ 'key':k,'model_id':SPECS[k]['model_id'],'revision':SPECS[k]['revision']} for k in KEYS],'transform':'clipped_logit','coefficients':model.coef_[0].tolist(),'intercept':float(model.intercept_[0]),'calibrated':False,'training_manifest':json.loads((a.scores/'dataset-manifest.json').read_text()),'validation':report} | |
| a.output.mkdir(exist_ok=True);(a.output/'pilot-fusion.json').write_text(json.dumps(artifact,indent=2));(a.output/'pilot-validation.json').write_text(json.dumps(report,indent=2)) | |
| print(json.dumps(report,indent=2)) | |
| if __name__=='__main__':main() | |