| """Sample public English source pairs with provenance; never use the uploaded paper.""" |
| import hashlib,json,urllib.request |
| from collections import Counter |
| from pathlib import Path |
| import ijson |
| from app.preprocessing import prepare |
| ROOT=Path(__file__).resolve().parents[1];out=ROOT/'work/research';out.mkdir(exist_ok=True) |
| meta=json.load(urllib.request.urlopen('https://huggingface.co/api/datasets/WUJUNCHAO/DetectRL-X')) |
| counts=Counter();seen=set();seen_texts=set();records=[] |
| url=f'https://huggingface.co/datasets/WUJUNCHAO/DetectRL-X/resolve/{meta["sha"]}/Binary/binary_general_open.json' |
| with urllib.request.urlopen(url,timeout=120) as stream: |
| for item in ijson.items(stream,'item'): |
| if item.get('lang')!='english':continue |
| human=item.get('human_written_text','');ai=item.get('llm_generated_text','');group=hashlib.sha256(human.encode()).hexdigest();stratum=(item['domain'],item['model']) |
| if counts[stratum]>=2 or group in seen or min(len(human.split()),len(ai.split()))<300:continue |
| normalized_human=prepare(' '.join(human.split()[:300]))['text'] |
| normalized_ai=prepare(' '.join(ai.split()[:300]))['text'] |
| fingerprints=[hashlib.sha256(t.encode()).hexdigest() for t in [normalized_human,normalized_ai]] |
| if len(set(fingerprints))!=2 or any(f in seen_texts for f in fingerprints):continue |
| counts[stratum]+=1;seen.add(group);seen_texts.update(fingerprints) |
| for text,label in [(normalized_human,'human'),(normalized_ai,'ai')]: |
| records.append({'text':text,'label':label,'group':group,'domain':item['domain'],'generator':item['model'],'dataset':'WUJUNCHAO/DetectRL-X','revision':meta['sha'],'source_split':item['split'],'selection':'first 300 words, same CP1 normalization as runtime; exact normalized excerpt deduplication across both classes'}) |
| if len(records)>=96:break |
| (out/'corpus.jsonl').write_text(''.join(json.dumps(r)+'\n' for r in records)) |
| manifest={'source':url,'revision':meta['sha'],'license_declared':meta.get('cardData',{}).get('license'),'normalization':'app.preprocessing.prepare, all scope','exact_normalized_duplicates':0,'documents':len(records),'source_groups':len(seen),'strata':{str(k):v for k,v in counts.items()},'limitations':['Convenience sample, not a private benchmark. Checkpoint training overlap unknown.','Split by human-source hash; author identities unavailable.','No edited human or non-native subgroup ground truth supplied.','Neither the uploaded paper nor commercial scores were used.']} |
| (out/'corpus-manifest.json').write_text(json.dumps(manifest,indent=2));print(json.dumps(manifest,indent=2)) |
|
|
