| import json,gc,time,traceback |
| from pathlib import Path |
| from app.detectors import Detector |
| root=Path('work/upgrade') |
| paper_path=Path('work/paper-analysis/normalized.txt') |
| paper=paper_path.read_text() if paper_path.exists() else None |
| for key in ['fakespot','earlybird','vanguard']: |
| print('Loading',key,flush=True) |
| try: |
| d=Detector(key);d.load() |
| if paper is not None: |
| report=json.loads((root/(key+'-paper.json')).read_text()) if (root/(key+'-paper.json')).exists() else d.analyze(paper) |
| (root/(key+'-paper.json')).write_text(json.dumps(report,indent=2)) |
| print('PAPER',key,report['ai_score'],report['duration_ms'],flush=True) |
| if not (root/'external.jsonl').exists(): |
| print('Corpus not ready',flush=True);continue |
| rows=[json.loads(l) for l in (root/'external.jsonl').read_text().splitlines()] |
| with (root/(key+'-external.jsonl')).open('w') as out: |
| for i,row in enumerate(rows): |
| r=d.analyze(' '.join(row['text'].split()[:300]));out.write(json.dumps({k:v for k,v in row.items() if k!='text'}|{'ai_score':r['ai_score'],'duration_ms':r['duration_ms'],'words':min(300,len(row['text'].split())),'excerpt_policy':'First 300 whitespace-delimited words for bounded-cost pilot; full original retained locally'})+'\n');out.flush() |
| if i%10==0:print(key,'controls',i+1,'/',len(rows),flush=True) |
| del d;gc.collect() |
| except Exception: |
| traceback.print_exc() |
|
|
