import urllib.request,json,ijson,hashlib
from pathlib import Path
mid='WUJUNCHAO/DetectRL-X'
meta=json.load(urllib.request.urlopen('https://huggingface.co/api/datasets/'+mid))
revision=meta['sha']
url=f'https://huggingface.co/datasets/{mid}/resolve/{revision}/Binary/binary_general_open.json'
seen=set();records=[]; strata=set()
with urllib.request.urlopen(url,timeout=120) as stream:
 for item in ijson.items(stream,'item'):
  if item.get('lang')!='english':continue
  h=item.get('human_written_text','');a=item.get('llm_generated_text','');group=hashlib.sha256(h.encode()).hexdigest()
  stratum=(item.get('domain'),item.get('model'))
  if stratum in strata:continue
  if group in seen or min(len(h.split()),len(a.split()))<80:continue
  seen.add(group);strata.add(stratum)
  for text,label in [(h,'human'),(a,'ai')]:records.append({'text':text,'label':label,'group':group,'domain':item.get('domain'),'generator':item.get('model'),'source_split':item.get('split'),'dataset':mid,'revision':revision})
  if len(records)>=48:break
out=Path('work/upgrade/external.jsonl');out.write_text(''.join(json.dumps(r)+'\n' for r in records))
Path('work/upgrade/dataset-manifest.json').write_text(json.dumps({'dataset':mid,'revision':revision,'file':'Binary/binary_general_open.json','license_declared':meta.get('cardData',{}).get('license'),'selection':'One distinct English human-source pair per domain/generator stratum, up to 24 pairs; both texts >=80 words; small convenience pilot, not representative. Paired groups retained across splits. Prior checkpoint training overlap unknown.','documents':len(records)},indent=2))
print('Selected',len(records),'documents',flush=True)