"""Shared offline decoding for the standalone releases.""" import re import torch from classifier import apply_threshold, batch, predict MARK_LABELS=["O","PERIOD","COMMA","QUESTION","COLON"] SYMBOLS=dict(zip(MARK_LABELS,["",".",",","?",":"])) def punct_encode(tokenizer,row,final_only=False): text=row["text"] words=list(re.finditer(r"\S+",text)) encoded=tokenizer(text,return_offsets_mapping=True,truncation=False,verbose=False) if len(encoded["input_ids"])>256: raise ValueError("Input exceeds 256 tokens including special tokens") ends={end:i for i,(start,end) in enumerate(encoded["offset_mapping"]) if start