Download scripts/train_research.py from Websterfly/Ai-detector-websterfly: direct link, hf CLI and curl.
- Browser
- Download file 8.18 kB
-
https://huggingface.co/spaces/Websterfly/Ai-detector-websterfly/resolve/main/scripts/train_research.py
- Command line
-
hf download hf://spaces/Websterfly/Ai-detector-websterfly/scripts/train_research.py
-
curl -L -o train_research.py https://huggingface.co/spaces/Websterfly/Ai-detector-websterfly/resolve/main/scripts/train_research.py
8.18 kB
| """Grouped train/selection/calibration/test research evaluation and head training.""" | |
| import hashlib,json,math | |
| from pathlib import Path | |
| import numpy as np | |
| from scipy.stats import binomtest | |
| from sklearn.linear_model import LogisticRegression | |
| from sklearn.preprocessing import StandardScaler | |
| from sklearn.feature_selection import SelectKBest,f_classif | |
| from sklearn.model_selection import GroupKFold | |
| from sklearn.metrics import roc_auc_score | |
| from app.research_model import ngram_train,ngram_features,linear_score,current_signature | |
| ROOT=Path(__file__).resolve().parents[1];D=ROOT/'work/research' | |
| def export_model(model,scaler,names): | |
| return {'features':list(names),'mean':scaler.mean_.tolist(),'scale':scaler.scale_.tolist(),'coefficients':model.coef_[0].tolist(),'intercept':float(model.intercept_[0])} | |
| def fit(X,y,names,weights=None,k=None): | |
| indices=np.arange(X.shape[1]) if k is None else SelectKBest(f_classif,k=min(k,X.shape[1])).fit(X,y).get_support(indices=True) | |
| scaler=StandardScaler().fit(X[:,indices]);model=LogisticRegression(C=.3,max_iter=2000).fit(scaler.transform(X[:,indices]),y,sample_weight=weights) | |
| return export_model(model,scaler,[names[i] for i in indices]) | |
| def stats(labels,scores,threshold): | |
| labels=np.array(labels);scores=np.array(scores);human=scores[labels==0];ai=scores[labels==1] | |
| fp=int((human>=threshold).sum());tp=int((ai>=threshold).sum());n=len(human) | |
| ci=binomtest(fp,n).proportion_ci(.95) if n else None | |
| return {'human_documents':n,'ai_documents':len(ai),'false_positives':fp,'true_positives':tp,'fpr':fp/n if n else None,'tpr':tp/len(ai) if len(ai) else None,'fpr_95pct_interval':[float(ci.low),float(ci.high)] if ci else None,'threshold':threshold,'auroc':float(roc_auc_score(labels,scores)) if len(set(labels))==2 else None} | |
| def main(): | |
| rows=[json.loads(l) for l in (D/'corpus.jsonl').read_text().splitlines()];splits=json.loads((D/'splits.json').read_text());lookup={} | |
| for kind in ['lm','meld','earlybird']: | |
| records=[json.loads(l) for l in (D/(kind+'-scores.jsonl')).read_text().splitlines()] | |
| lookup[kind]={r['input_sha256']:r['result'] for r in records} | |
| ids=[hashlib.sha256(r['text'].encode()).hexdigest() for r in rows] | |
| if len(ids)!=len(set(ids)):raise ValueError('Duplicate normalized input text across corpus records; evaluation stopped.') | |
| from app.preprocessing import prepare | |
| if any(prepare(r['text'])['text']!=r['text'] for r in rows):raise ValueError('Training/runtime preprocessing mismatch; evaluation stopped.') | |
| if any(any(i not in lookup[k] for k in lookup) for i in ids):raise ValueError('Incomplete expert scores; evaluation stopped.') | |
| y=np.array([r['label']=='ai' for r in rows],int);groups=np.array([r['group'] for r in rows]);part=np.array([splits[g] for g in groups]);train=np.where(part=='train')[0];selection=np.where(part=='selection')[0];cal=np.where(part=='calibration')[0];test=np.where(part=='test')[0] | |
| ng=ngram_train([rows[i]['text'] for i in train if y[i]==0]) | |
| feature_dicts=[lookup['lm'][i]['features']|ngram_features(r['text'],ng) for i,r in zip(ids,rows)] | |
| names=sorted(feature_dicts[0]);X=np.array([[f[n] for n in names] for f in feature_dicts]) | |
| best=None;best_auc=-1;weights=np.ones(len(train));history=[] | |
| for round_id in range(3): | |
| model=fit(X[train],y[train],names,weights,k=8) | |
| pred=np.array([linear_score(f,model) for f in feature_dicts]);auc=roc_auc_score(y[selection],pred[selection]);history.append({'round':round_id,'selection_auroc':float(auc),'weighted_pairs':int((weights>1).sum()/2)}) | |
| if auc>best_auc:best=model;best_auc=auc | |
| hard_groups={groups[i] for i in train if (pred[i]>=.5)!=bool(y[i])} | |
| weights=np.array([2.0 if groups[i] in hard_groups else 1.0 for i in train]) | |
| ghost=np.array([linear_score(f,best) for f in feature_dicts]);ghost_oof=np.zeros(len(train)) | |
| # Rebuild ngram profiles inside each OOF fold too; no held-out fold text enters its features. | |
| for tr,va in GroupKFold(4).split(X[train],y[train],groups[train]): | |
| profile=ngram_train([rows[train[i]]['text'] for i in tr if y[train[i]]==0]) | |
| fs=[lookup['lm'][ids[train[i]]]['features']|ngram_features(rows[train[i]]['text'],profile) for i in range(len(train))] | |
| xx=np.array([[f[n] for n in names] for f in fs]);m=fit(xx[tr],y[train][tr],names,k=8) | |
| ghost_oof[va]=[linear_score(fs[i],m) for i in va] | |
| early=np.array([lookup['earlybird'][i]['ai_score'] for i in ids]);meld=np.array([lookup['meld'][i]['score'] for i in ids]);bino=np.array([f['binoculars_neg_ratio'] for f in feature_dicts]);fast=np.array([f['fast_discrepancy'] for f in feature_dicts]) | |
| meta_names=['ghostbuster_inspired','earlybird','meld','binoculars_neg_ratio','fast_discrepancy'] | |
| meta=np.column_stack([ghost,early,meld,bino,fast]);meta_train=meta[train].copy();meta_train[:,0]=ghost_oof | |
| fused_model=fit(meta_train,y[train],meta_names) | |
| fused=np.array([linear_score(dict(zip(meta_names,v)),fused_model) for v in meta]) | |
| scores={'earlybird':early,'meld_variant':meld,'binoculars_proxy':bino,'fast_detectgpt_proxy':fast,'ghostbuster_inspired':ghost,'fusion':fused} | |
| thresholds={};evaluations={};subgroups={} | |
| for name,values in scores.items(): | |
| human=values[cal][y[cal]==0];thresholds[name]={};evaluations[name]={} | |
| for alpha in [.01,.001,.0001]: | |
| # Empirical human quantile, always fitted on calibration only. Small n cannot validate tail claims. | |
| t=float(np.nextafter(np.quantile(human,1-alpha,method='higher'),np.inf));thresholds[name][str(alpha)]=t | |
| evaluations[name][str(alpha)]=stats(y[test],values[test],t)|{'target_fpr':alpha,'calibration_human_documents':len(human),'tail_supported':False} | |
| subgroups[name]={} | |
| for field in ['domain','generator']: | |
| for value in sorted({rows[i][field] for i in test}): | |
| chosen=[i for i in test if rows[i][field]==value] | |
| subgroups[name][field+':'+value]=stats(y[chosen],values[chosen],thresholds[name]['0.01']) | |
| artifact={'schema':1,'signature':current_signature(),'ghost':best,'fusion':fused_model,'ngram':ng,'thresholds':thresholds,'training_groups':len(set(groups[train])),'release_validated':False,'base_variants':{'MELD':json.loads((ROOT/'artifacts/meld/provenance.json').read_text()),'LM':'GPT2/DistilGPT2'},'selection_history':history} | |
| (ROOT/'artifacts/research-heads.json').write_text(json.dumps(artifact)) | |
| report={'status':'pilot_not_release_validated','documents':len(rows),'partitions':{k:int((part==k).sum()) for k in set(part)},'evaluations':evaluations,'subgroups':subgroups,'hard_negative_history':history, | |
| 'limitations':['Small public convenience corpus; pretrained model overlap unknown.','Only eight human calibration and eight human test documents; no claimed 1%, 0.1%, or 0.01% FPR guarantee.','All thresholds use calibration labels only. The test set was evaluated after fitting.','Only first-300-word excerpts; whole-paper behavior not validated.','No human-polished or non-native subgroup ground truth; no fairness or mixed-proportion accuracy claim.','MELD uses derived BF16 storage/backbone inference with FP32 detector heads.','Ghostbuster-inspired classifier replaces the original scorers; it is a new research model.','Hard-negative loop is training-pool pair reweighting, not large-pool fresh mirror generation.']} | |
| baseline=evaluations['meld_variant']['0.01'];combined=evaluations['fusion']['0.01'] | |
| report['pilot_conclusion']={'leading_candidate':'unselected_research_candidates','fusion_promoted':False, | |
| 'reason':f"Pilot only: MELD {baseline['true_positives']}/{baseline['ai_documents']} AI with {baseline['false_positives']} human false positives; fusion {combined['true_positives']}/{combined['ai_documents']} AI with {combined['false_positives']} human false positives. No promotion from this small public corpus.", | |
| 'next_gate':'A larger fresh independent holdout, not post-hoc threshold adjustment.'} | |
| (ROOT/'artifacts/research-validation.json').write_text(json.dumps(report,indent=2));print(json.dumps(report,indent=2)) | |
| if __name__=='__main__':main() | |