| import urllib.request,json,ijson,hashlib |
| from pathlib import Path |
| mid='WUJUNCHAO/DetectRL-X' |
| meta=json.load(urllib.request.urlopen('https://huggingface.co/api/datasets/'+mid)) |
| revision=meta['sha'] |
| url=f'https://huggingface.co/datasets/{mid}/resolve/{revision}/Binary/binary_general_open.json' |
| seen=set();records=[]; strata=set() |
| with urllib.request.urlopen(url,timeout=120) as stream: |
| for item in ijson.items(stream,'item'): |
| if item.get('lang')!='english':continue |
| h=item.get('human_written_text','');a=item.get('llm_generated_text','');group=hashlib.sha256(h.encode()).hexdigest() |
| stratum=(item.get('domain'),item.get('model')) |
| if stratum in strata:continue |
| if group in seen or min(len(h.split()),len(a.split()))<80:continue |
| seen.add(group);strata.add(stratum) |
| for text,label in [(h,'human'),(a,'ai')]:records.append({'text':text,'label':label,'group':group,'domain':item.get('domain'),'generator':item.get('model'),'source_split':item.get('split'),'dataset':mid,'revision':revision}) |
| if len(records)>=48:break |
| out=Path('work/upgrade/external.jsonl');out.write_text(''.join(json.dumps(r)+'\n' for r in records)) |
| Path('work/upgrade/dataset-manifest.json').write_text(json.dumps({'dataset':mid,'revision':revision,'file':'Binary/binary_general_open.json','license_declared':meta.get('cardData',{}).get('license'),'selection':'One distinct English human-source pair per domain/generator stratum, up to 24 pairs; both texts >=80 words; small convenience pilot, not representative. Paired groups retained across splits. Prior checkpoint training overlap unknown.','documents':len(records)},indent=2)) |
| print('Selected',len(records),'documents',flush=True) |
|
|
