records.py

"""Direct UTF-8 bytes. Special IDs match BET, not the old Cortex tokenizer."""
from dataclasses import dataclass
PAD,BOS,EOS=256,257,258

class Oversize(ValueError):pass
class InvalidRecord(ValueError):pass

def ids(text):return list(text.encode('utf-8'))
def decode(tokens):return bytes(t for t in tokens if 0<=t<256).decode('utf-8',errors='replace')

def record(prefix,answer,source,limit=1024,supervise_all=False,meta=None):
    p,a=ids(prefix),ids(answer)
    if not a:raise InvalidRecord('Empty target: '+source)
    tokens=[BOS]+p+a+[EOS]
    if len(tokens)>limit+1:raise Oversize(f'{source}: {len(tokens)} IDs exceeds {limit+1}; no truncation')
    weights=[0]+([1]*len(p) if supervise_all else [0]*len(p))+[1]*(len(a)+1)
    return dict(ids=tokens,weights=weights,source=source,prompt_len=1+len(p),meta=meta or {})

def plain_chunks(text,source,limit=1024):
    # Lossless bytes, including split UTF-8 sequences: decoder assembles the byte stream.
    # No false EOS at chunk boundaries. One-token overlap predicts each byte once.
    if not isinstance(text,str) or not text.strip():raise InvalidRecord('Empty/non-string text: '+source)
    raw_bytes=text.encode('utf-8')
    if len(raw_bytes)>1024*1024:raise Oversize('Document exceeds the 1 MiB bounded-buffer limit; rejected intact')
    raw=[BOS]+list(raw_bytes)+[EOS];out=[]
    for offset in range(0,len(raw)-1,limit):
        chunk=raw[offset:offset+limit+1]
        out.append(dict(ids=chunk,weights=[0]+[1]*(len(chunk)-1),source=source,prompt_len=1,meta={}))
    return out

def validate(r,limit=1024):
    assert 2<=len(r['ids'])<=limit+1
    assert len(r['ids'])==len(r['weights'])
    assert all(type(x)==int and 0<=x<259 for x in r['ids'])
    assert all(x in (0,1) for x in r['weights']) and sum(r['weights'][1:])>0
    assert r['weights'][0]==0
    return r