Ai-detector-websterfly / scripts /train_research.py
Websterfly's picture
Deploy Websterfly educational AI detection workbench
97799b9 verified
Raw History Blame Contribute Delete
8.18 kB
"""Grouped train/selection/calibration/test research evaluation and head training."""
import hashlib,json,math
from pathlib import Path
import numpy as np
from scipy.stats import binomtest
from sklearn.linear_model import LogisticRegression
from sklearn.preprocessing import StandardScaler
from sklearn.feature_selection import SelectKBest,f_classif
from sklearn.model_selection import GroupKFold
from sklearn.metrics import roc_auc_score
from app.research_model import ngram_train,ngram_features,linear_score,current_signature
ROOT=Path(__file__).resolve().parents[1];D=ROOT/'work/research'
def export_model(model,scaler,names):
return {'features':list(names),'mean':scaler.mean_.tolist(),'scale':scaler.scale_.tolist(),'coefficients':model.coef_[0].tolist(),'intercept':float(model.intercept_[0])}
def fit(X,y,names,weights=None,k=None):
indices=np.arange(X.shape[1]) if k is None else SelectKBest(f_classif,k=min(k,X.shape[1])).fit(X,y).get_support(indices=True)
scaler=StandardScaler().fit(X[:,indices]);model=LogisticRegression(C=.3,max_iter=2000).fit(scaler.transform(X[:,indices]),y,sample_weight=weights)
return export_model(model,scaler,[names[i] for i in indices])
def stats(labels,scores,threshold):
labels=np.array(labels);scores=np.array(scores);human=scores[labels==0];ai=scores[labels==1]
fp=int((human>=threshold).sum());tp=int((ai>=threshold).sum());n=len(human)
ci=binomtest(fp,n).proportion_ci(.95) if n else None
return {'human_documents':n,'ai_documents':len(ai),'false_positives':fp,'true_positives':tp,'fpr':fp/n if n else None,'tpr':tp/len(ai) if len(ai) else None,'fpr_95pct_interval':[float(ci.low),float(ci.high)] if ci else None,'threshold':threshold,'auroc':float(roc_auc_score(labels,scores)) if len(set(labels))==2 else None}
def main():
rows=[json.loads(l) for l in (D/'corpus.jsonl').read_text().splitlines()];splits=json.loads((D/'splits.json').read_text());lookup={}
for kind in ['lm','meld','earlybird']:
records=[json.loads(l) for l in (D/(kind+'-scores.jsonl')).read_text().splitlines()]
lookup[kind]={r['input_sha256']:r['result'] for r in records}
ids=[hashlib.sha256(r['text'].encode()).hexdigest() for r in rows]
if len(ids)!=len(set(ids)):raise ValueError('Duplicate normalized input text across corpus records; evaluation stopped.')
from app.preprocessing import prepare
if any(prepare(r['text'])['text']!=r['text'] for r in rows):raise ValueError('Training/runtime preprocessing mismatch; evaluation stopped.')
if any(any(i not in lookup[k] for k in lookup) for i in ids):raise ValueError('Incomplete expert scores; evaluation stopped.')
y=np.array([r['label']=='ai' for r in rows],int);groups=np.array([r['group'] for r in rows]);part=np.array([splits[g] for g in groups]);train=np.where(part=='train')[0];selection=np.where(part=='selection')[0];cal=np.where(part=='calibration')[0];test=np.where(part=='test')[0]
ng=ngram_train([rows[i]['text'] for i in train if y[i]==0])
feature_dicts=[lookup['lm'][i]['features']|ngram_features(r['text'],ng) for i,r in zip(ids,rows)]
names=sorted(feature_dicts[0]);X=np.array([[f[n] for n in names] for f in feature_dicts])
best=None;best_auc=-1;weights=np.ones(len(train));history=[]
for round_id in range(3):
model=fit(X[train],y[train],names,weights,k=8)
pred=np.array([linear_score(f,model) for f in feature_dicts]);auc=roc_auc_score(y[selection],pred[selection]);history.append({'round':round_id,'selection_auroc':float(auc),'weighted_pairs':int((weights>1).sum()/2)})
if auc>best_auc:best=model;best_auc=auc
hard_groups={groups[i] for i in train if (pred[i]>=.5)!=bool(y[i])}
weights=np.array([2.0 if groups[i] in hard_groups else 1.0 for i in train])
ghost=np.array([linear_score(f,best) for f in feature_dicts]);ghost_oof=np.zeros(len(train))
# Rebuild ngram profiles inside each OOF fold too; no held-out fold text enters its features.
for tr,va in GroupKFold(4).split(X[train],y[train],groups[train]):
profile=ngram_train([rows[train[i]]['text'] for i in tr if y[train[i]]==0])
fs=[lookup['lm'][ids[train[i]]]['features']|ngram_features(rows[train[i]]['text'],profile) for i in range(len(train))]
xx=np.array([[f[n] for n in names] for f in fs]);m=fit(xx[tr],y[train][tr],names,k=8)
ghost_oof[va]=[linear_score(fs[i],m) for i in va]
early=np.array([lookup['earlybird'][i]['ai_score'] for i in ids]);meld=np.array([lookup['meld'][i]['score'] for i in ids]);bino=np.array([f['binoculars_neg_ratio'] for f in feature_dicts]);fast=np.array([f['fast_discrepancy'] for f in feature_dicts])
meta_names=['ghostbuster_inspired','earlybird','meld','binoculars_neg_ratio','fast_discrepancy']
meta=np.column_stack([ghost,early,meld,bino,fast]);meta_train=meta[train].copy();meta_train[:,0]=ghost_oof
fused_model=fit(meta_train,y[train],meta_names)
fused=np.array([linear_score(dict(zip(meta_names,v)),fused_model) for v in meta])
scores={'earlybird':early,'meld_variant':meld,'binoculars_proxy':bino,'fast_detectgpt_proxy':fast,'ghostbuster_inspired':ghost,'fusion':fused}
thresholds={};evaluations={};subgroups={}
for name,values in scores.items():
human=values[cal][y[cal]==0];thresholds[name]={};evaluations[name]={}
for alpha in [.01,.001,.0001]:
# Empirical human quantile, always fitted on calibration only. Small n cannot validate tail claims.
t=float(np.nextafter(np.quantile(human,1-alpha,method='higher'),np.inf));thresholds[name][str(alpha)]=t
evaluations[name][str(alpha)]=stats(y[test],values[test],t)|{'target_fpr':alpha,'calibration_human_documents':len(human),'tail_supported':False}
subgroups[name]={}
for field in ['domain','generator']:
for value in sorted({rows[i][field] for i in test}):
chosen=[i for i in test if rows[i][field]==value]
subgroups[name][field+':'+value]=stats(y[chosen],values[chosen],thresholds[name]['0.01'])
artifact={'schema':1,'signature':current_signature(),'ghost':best,'fusion':fused_model,'ngram':ng,'thresholds':thresholds,'training_groups':len(set(groups[train])),'release_validated':False,'base_variants':{'MELD':json.loads((ROOT/'artifacts/meld/provenance.json').read_text()),'LM':'GPT2/DistilGPT2'},'selection_history':history}
(ROOT/'artifacts/research-heads.json').write_text(json.dumps(artifact))
report={'status':'pilot_not_release_validated','documents':len(rows),'partitions':{k:int((part==k).sum()) for k in set(part)},'evaluations':evaluations,'subgroups':subgroups,'hard_negative_history':history,
'limitations':['Small public convenience corpus; pretrained model overlap unknown.','Only eight human calibration and eight human test documents; no claimed 1%, 0.1%, or 0.01% FPR guarantee.','All thresholds use calibration labels only. The test set was evaluated after fitting.','Only first-300-word excerpts; whole-paper behavior not validated.','No human-polished or non-native subgroup ground truth; no fairness or mixed-proportion accuracy claim.','MELD uses derived BF16 storage/backbone inference with FP32 detector heads.','Ghostbuster-inspired classifier replaces the original scorers; it is a new research model.','Hard-negative loop is training-pool pair reweighting, not large-pool fresh mirror generation.']}
baseline=evaluations['meld_variant']['0.01'];combined=evaluations['fusion']['0.01']
report['pilot_conclusion']={'leading_candidate':'unselected_research_candidates','fusion_promoted':False,
'reason':f"Pilot only: MELD {baseline['true_positives']}/{baseline['ai_documents']} AI with {baseline['false_positives']} human false positives; fusion {combined['true_positives']}/{combined['ai_documents']} AI with {combined['false_positives']} human false positives. No promotion from this small public corpus.",
'next_gate':'A larger fresh independent holdout, not post-hoc threshold adjustment.'}
(ROOT/'artifacts/research-validation.json').write_text(json.dumps(report,indent=2));print(json.dumps(report,indent=2))
if __name__=='__main__':main()