File size: 1,752 Bytes
ceaaf21 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 | """Validate published tokenized JSONL and build a local PyTorch cache."""
import argparse
import gzip
import json
import math
from pathlib import Path
import torch
def load_rows(path):
source=Path(path)
opener=gzip.open if source.suffix=='.gz' else open
rows=[]
with opener(source,'rt',encoding='utf-8') as stream:
for number,line in enumerate(stream,1):
row=json.loads(line)
ids,markers,target=row['ids'],row['markers'],row['target']
if not ids or any(type(x) is not int or x<0 for x in ids):
raise ValueError(f'line {number}: invalid token IDs')
if len(markers)!=len(target) or len(target)<2 or any(type(x) is not int or not 0<=x<len(ids) for x in markers):
raise ValueError(f'line {number}: invalid markers')
if any(type(p) not in (int,float) or not math.isfinite(p) or p<0 for p in target) or not math.isclose(sum(target),1,abs_tol=1e-5):
raise ValueError(f'line {number}: invalid probabilities')
if row['qtype']!={'choice':0,'score':1,'noul':2}.get(row['t']):
raise ValueError(f'line {number}: invalid question type')
rows.append(row)
if not rows:
raise ValueError('Empty dataset')
return rows
def main():
parser=argparse.ArgumentParser()
parser.add_argument('--input',required=True)
parser.add_argument('--output',required=True)
args=parser.parse_args()
output=Path(args.output)
if output.exists():
raise FileExistsError(output)
rows=load_rows(args.input)
output.parent.mkdir(parents=True,exist_ok=True)
torch.save(rows,output)
print(f'Validated {len(rows)} records')
if __name__=='__main__':
main()
|