File size: 2,555 Bytes
97799b9 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 | """Sample public English source pairs with provenance; never use the uploaded paper."""
import hashlib,json,urllib.request
from collections import Counter
from pathlib import Path
import ijson
from app.preprocessing import prepare
ROOT=Path(__file__).resolve().parents[1];out=ROOT/'work/research';out.mkdir(exist_ok=True)
meta=json.load(urllib.request.urlopen('https://huggingface.co/api/datasets/WUJUNCHAO/DetectRL-X'))
counts=Counter();seen=set();seen_texts=set();records=[]
url=f'https://huggingface.co/datasets/WUJUNCHAO/DetectRL-X/resolve/{meta["sha"]}/Binary/binary_general_open.json'
with urllib.request.urlopen(url,timeout=120) as stream:
for item in ijson.items(stream,'item'):
if item.get('lang')!='english':continue
human=item.get('human_written_text','');ai=item.get('llm_generated_text','');group=hashlib.sha256(human.encode()).hexdigest();stratum=(item['domain'],item['model'])
if counts[stratum]>=2 or group in seen or min(len(human.split()),len(ai.split()))<300:continue
normalized_human=prepare(' '.join(human.split()[:300]))['text']
normalized_ai=prepare(' '.join(ai.split()[:300]))['text']
fingerprints=[hashlib.sha256(t.encode()).hexdigest() for t in [normalized_human,normalized_ai]]
if len(set(fingerprints))!=2 or any(f in seen_texts for f in fingerprints):continue
counts[stratum]+=1;seen.add(group);seen_texts.update(fingerprints)
for text,label in [(normalized_human,'human'),(normalized_ai,'ai')]:
records.append({'text':text,'label':label,'group':group,'domain':item['domain'],'generator':item['model'],'dataset':'WUJUNCHAO/DetectRL-X','revision':meta['sha'],'source_split':item['split'],'selection':'first 300 words, same CP1 normalization as runtime; exact normalized excerpt deduplication across both classes'})
if len(records)>=96:break
(out/'corpus.jsonl').write_text(''.join(json.dumps(r)+'\n' for r in records))
manifest={'source':url,'revision':meta['sha'],'license_declared':meta.get('cardData',{}).get('license'),'normalization':'app.preprocessing.prepare, all scope','exact_normalized_duplicates':0,'documents':len(records),'source_groups':len(seen),'strata':{str(k):v for k,v in counts.items()},'limitations':['Convenience sample, not a private benchmark. Checkpoint training overlap unknown.','Split by human-source hash; author identities unavailable.','No edited human or non-native subgroup ground truth supplied.','Neither the uploaded paper nor commercial scores were used.']}
(out/'corpus-manifest.json').write_text(json.dumps(manifest,indent=2));print(json.dumps(manifest,indent=2))
|