RealArDor/NanoJev-SimuNRGTrans / source /scripts /merge_decision_datasets.py
RealArDor's picture
download
raw
2.1 kB
#!/usr/bin/env python3
"""Merge already split local datasets without changing their split assignments."""
import argparse
from collections import Counter
import hashlib
import json
from pathlib import Path
def merge(inputs, output):
rows, seen, groups, sources = [], set(), {}, []
for name in inputs:
file = Path(name); raw = file.read_bytes()
sources.append({'path':str(file), 'sha256':hashlib.sha256(raw).hexdigest()})
for line in raw.decode().splitlines():
if not line.strip(): continue
row = json.loads(line)
if row['id'] in seen: raise ValueError('Duplicate record ID')
seen.add(row['id'])
group = row['metadata']['source_group_id']
if group in groups and groups[group] != row['split']: raise ValueError('Cross-split source group')
groups[group] = row['split']
rows.append(row)
order = {s:i for i,s in enumerate(['train','dev','calibration','test','ood'])}
rows.sort(key=lambda r:(order[r['split']],r['family_id'],r['id']))
text = ''.join(json.dumps(r,ensure_ascii=False)+'\n' for r in rows)
output = Path(output); output.parent.mkdir(parents=True,exist_ok=True);output.write_text(text)
manifest = {'schema_version':'openjev-merged-v2','sources':sources,'output_sha256':hashlib.sha256(text.encode()).hexdigest(),
'states':len(rows),'questions':sum(len(r['questions']) for r in rows),'source_groups':len(groups),
'split_states':dict(Counter(r['split'] for r in rows)),
'split_family_states':dict(Counter(f"{r['split']}/{r['family_id']}" for r in rows)),
'source_groups_overlap_between_splits':False, 'training_split_changed':False}
output.with_suffix('.manifest.json').write_text(json.dumps(manifest,indent=2)+'\n')
return manifest
if __name__ == '__main__':
p=argparse.ArgumentParser(description=__doc__);p.add_argument('--inputs',nargs='+',required=True);p.add_argument('--output',required=True)
a=p.parse_args();print(json.dumps(merge(a.inputs,a.output),indent=2))

Xet Storage Details

Size:
2.1 kB
·
Xet hash:
8ff871c5d814d02668b239ab7ad8180fdb0e5c2c7ed229eff1a6f177c7da9dc8

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.