File size: 7,114 Bytes
20fc05e
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
"""Constrained decoding.  On the main line no structure token and no </ids> can come, and <ids> only between
characters.  Inside an IDS every step keeps it well formed: after <ids> a structure token; a structure token opens as many
places as it has parts; a part fills one place and is one CJK character, radical or stroke (base-vocabulary tokens are
followed byte by byte through UTF-8); </ids> comes exactly when no place is open.  told(): what the model is told of an
IDS so far -- a structure token and the unfinished bytes of a part stand for themselves, whatever finishes a part is
told as </ids>."""
from __future__ import annotations
import numpy as np
import mlx.core as mx

NEG=-1e9
LEAVES=((0x4E00,0x9FFF),(0x3400,0x4DBF),(0x20000,0x3347F),(0x2E80,0x2EFF),(0x31C0,0x31E5))
BASE,SPECIAL,GLYPH,EXTCHAR,KOPEN,KCLOSE,STRUCT=range(7)
def leaf(cp):
    cp=np.asarray(cp);ok=np.zeros(cp.shape,bool)
    for a,b in LEAVES:ok|=(cp>=a)&(cp<=b)
    return ok
def byte_decoder():
    bs=list(range(ord('!'),ord('~')+1))+list(range(ord('¡'),ord('¬')+1))+list(range(ord('®'),ord('ÿ')+1));cs=bs[:];n=0
    for b in range(256):
        if b not in bs:bs.append(b);cs.append(256+n);n+=1
    return {chr(c):b for b,c in zip(bs,cs)}
def char_length(first):
    return 1 if first<0x80 else 2 if 0xC0<=first<=0xDF else 3 if 0xE0<=first<=0xEF else 4 if 0xF0<=first<=0xF7 else 0
def code(seq):
    cp=seq[0]&{1:0x7F,2:0x1F,3:0x0F,4:0x07}[char_length(seq[0])]
    for b in seq[1:]:cp=(cp<<6)|(b&0x3F)
    return cp
def partial_can_be_leaf(tail):
    n=char_length(tail[0])
    if n<3 or any(not 0x80<=b<=0xBF for b in tail[1:]):return False
    low=code(list(tail)+[0x80]*(n-len(tail)));high=code(list(tail)+[0xBF]*(n-len(tail)))
    return any(a<=high and low<=b for a,b in LEAVES)
def shape(data):
    i=0
    while i<len(data) and 0x80<=data[i]<=0xBF:i+=1
    lead=i;count=0;ok=True;trail=0
    while i<len(data):
        n=char_length(data[i])
        if n==0:return lead,count,0,False
        if i+n>len(data):trail=n-(len(data)-i);ok=ok and partial_can_be_leaf(data[i:]);break
        piece=data[i:i+n]
        if any(not 0x80<=b<=0xBF for b in piece[1:]):return lead,count,0,False
        ok=ok and bool(leaf(code(list(piece))));count+=1;i+=n
    return lead,count,trail,ok
class Vocab:
    def __init__(self,tokenizer,info):
        ids=info['ids'];V=ids['vocabulary'];self.V=V;self.open=ids['open'];self.close=ids['close'];self.standin=info['first_gw_id'];self.first_ids=ids['first_ids']
        self.kind=np.zeros(V,np.int8);self.lead=np.zeros(V,np.int8);self.chars=np.zeros(V,np.int16)
        self.trail=np.zeros(V,np.int8);self.ok=np.zeros(V,bool);self.arity=np.zeros(V,np.int8);self.data=[b'']*V
        self.first=np.full((V,3),-1,np.int16)
        specials={i for i in tokenizer.added_tokens_decoder if i<info['offset']}
        decoder=byte_decoder();pieces=tokenizer.convert_ids_to_tokens(list(range(info['offset'])))
        for i,piece in enumerate(pieces):
            if i in specials or piece is None or any(c not in decoder for c in piece):self.kind[i]=SPECIAL;continue
            data=bytes(decoder[c] for c in piece);self.data[i]=data
            self.lead[i],self.chars[i],self.trail[i],self.ok[i]=shape(data)
            for j,b in enumerate(data[:min(3,self.lead[i])]):self.first[i,j]=b
        self.kind[info['offset']:info['first_extension_id']]=GLYPH;self.kind[info['first_gw_id']:ids['first_ids']]=GLYPH
        self.kind[info['first_extension_id']:info['first_gw_id']]=EXTCHAR
        for i,piece in zip(range(info['first_extension_id'],info['first_gw_id']),
                           tokenizer.convert_ids_to_tokens(list(range(info['first_extension_id'],info['first_gw_id'])))):
            self.ok[i]=len(piece)==1 and bool(leaf(ord(piece)))
        self.kind[self.open]=KOPEN;self.kind[self.close]=KCLOSE
        for i,a in ids['arity'].items():self.kind[int(i)]=STRUCT;self.arity[int(i)]=a
        outside=~np.isin(self.kind,[KCLOSE,STRUCT])
        self.outside_np={True:outside,False:outside&(self.kind!=KOPEN)}
        self.outside={k:mx.array(v) for k,v in self.outside_np.items()};self.structures=self.kind==STRUCT
        self.continuation_only=np.where((self.kind==BASE)&(self.chars==0)&(self.trail==0)&(self.lead>0)
                                        &(self.lead==np.array([len(d) for d in self.data])))[0]
    def inside(self,need,pending,partial):
        m=np.zeros(self.V,bool)
        if need==0:m[self.close]=True;return m
        if pending==0:
            k=self.chars+(self.trail>0)
            m=(self.kind==STRUCT)|((self.kind==EXTCHAR)&self.ok)|((self.kind==BASE)&(self.lead==0)&self.ok&(k>=1)&(k<=need))
            return m
        finish=np.where((self.kind==BASE)&(self.lead==pending))[0]
        if len(finish):
            tail=self.first[finish,:pending].astype(np.int64);cp=code(list(partial))
            for j in range(pending):cp=(cp<<6)|(tail[:,j]&0x3F)
            k=1+self.chars[finish]+(self.trail[finish]>0)
            m[finish]=leaf(cp)&self.ok[finish]&(k<=need)
        for i in self.continuation_only:
            if self.lead[i]<pending and partial_can_be_leaf(partial+self.data[i]):m[i]=True
        return m
def advance(pending,partial,data):
    done=0;i=0
    if pending:
        take=min(pending,len(data));partial=partial+data[:take];pending-=take;i=take
        if pending==0:done+=1;partial=b''
    while i<len(data):
        n=char_length(data[i]) or 1
        if i+n>len(data):pending=n-(len(data)-i);partial=data[i:];break
        done+=1;i+=n
    return done,pending,partial
class Constraint:
    def __init__(self,vocab,may_open=True):
        self.v=vocab;self.may_open=may_open;self.primed=False;self.inside=False;self.first=False;self.need=0;self.pending=0;self.partial=b'';self.dead_ends=0
    def feed(self,t):
        v=self.v;k=v.kind[t]
        if not self.inside:
            if k==KOPEN:self.inside=True;self.first=True;self.need=1;self.pending=0;self.partial=b''
            elif k==BASE:_,self.pending,self.partial=advance(self.pending,self.partial,v.data[t])
            else:self.pending=0;self.partial=b''
            return
        self.first=False
        if k==KCLOSE:self.inside=False
        elif k==STRUCT:self.need+=int(v.arity[t])-1
        elif k==EXTCHAR:self.need-=1
        elif k==BASE:
            done,self.pending,self.partial=advance(self.pending,self.partial,v.data[t]);self.need-=done
    def allowed(self):
        if not self.inside:return self.v.outside_np[self.may_open and self.pending==0]
        if self.first:return self.v.structures
        return self.v.inside(self.need,self.pending,self.partial)
    def __call__(self,tokens,logits):
        if self.primed:self.feed(int(tokens[-1].item()))
        self.primed=True
        if not self.inside:return mx.where(self.v.outside[self.may_open and self.pending==0][None],logits,NEG)
        m=self.allowed()
        if not m.any():self.dead_ends+=1;return logits
        return mx.where(mx.array(m)[None],logits,NEG)
def told(vocab,constraint,t):
    return int(t) if vocab.kind[t]==STRUCT or constraint.pending else int(vocab.close)