import json,random from encode import encode_document, batch_to_device from structure import make_structure def rows(path,seed=None,limit=0): offsets=[] with open(path,'rb') as f: while True: pos=f.tell() if not f.readline():break offsets.append(pos) if seed is not None:random.Random(seed).shuffle(offsets) with open(path,'rb') as f: for i,pos in enumerate(offsets): if limit and i>=limit:break f.seek(pos) row=json.loads(f.readline()) s,e=row['body_span'];text=row['text'] if not isinstance(s,int) or not isinstance(e,int) or not 0<=s