"""Compute frozen expert features on a fixed, paired corpus. No paper is used."""
import hashlib,json,time
from pathlib import Path
import numpy as np
import torch
from app.experts import LMExperts,MeldExpert
from app.detectors import Detector
ROOT=Path(__file__).resolve().parents[1];OUT=ROOT/'work/research';torch.set_num_threads(2)
rows=[json.loads(l) for l in (OUT/'corpus.jsonl').read_text().splitlines()]
groups=sorted(set(r['group'] for r in rows));np.random.default_rng(1847).shuffle(groups)
splits={g:'train' if i<24 else 'selection' if i<32 else 'calibration' if i<40 else 'test' for i,g in enumerate(groups)}
(OUT/'splits.json').write_text(json.dumps(splits,indent=2))
for kind in ['earlybird','lm','meld']:
 path=OUT/(kind+'-scores.jsonl');done={}
 if path.exists():
  for line in path.read_text().splitlines():
   r=json.loads(line);done[r['input_sha256']]=r
 engine=Detector('earlybird') if kind=='earlybird' else LMExperts() if kind=='lm' else MeldExpert()
 if kind=='earlybird':engine.load()
 with path.open('a') as out:
  for i,row in enumerate(rows):
   sha=hashlib.sha256(row['text'].encode()).hexdigest()
   if sha in done:continue
   begin=time.time();result=engine.features(row['text']) if kind=='lm' else engine.analyze(row['text'])
   record={k:v for k,v in row.items() if k!='text'}|{'input_sha256':sha,'split':splits[row['group']],'result':result,'seconds':time.time()-begin}
   out.write(json.dumps(record)+'\n');out.flush()
   if i%8==0:print(kind,i+1,'/',len(rows),'seconds',round(record['seconds'],2),flush=True)
 if kind!='earlybird':engine.unload()
 del engine
 import gc;gc.collect()