astromandarin's picture
OpenGlyph-OCR-IDS: first release
20fc05e verified
Raw History Blame Contribute Delete
4.07 kB
"""The IDS layer on top of OpenGlyph-OCR-0: fourteen added tokens (<ids>, </ids> and the twelve structure characters
⿰⿱⿲⿳⿴⿵⿶⿷⿸⿹⿺⿻) with fixed input rows -- rows the frozen layers know as "a character was begun and is not finished" --
and output rows and biases of their own; and the swapping of one set of weights for another for a while (reading_as).
Four sets of weights work in the one model: the reader (OpenGlyph-OCR-0's own, never changed), which writes the main
line; the opener, which says where an IDS begins; the part reader, which reads the cells of an unfolded glyph; the
speller, which spells an IDS from the glyph as it stands. The model carries the opener's weights; the others are put in
for a while and taken out again."""
from __future__ import annotations
from contextlib import contextmanager
import mlx.core as mx
import mlx.nn as nn
from mlx.utils import tree_flatten
from .vocab import extend_tokenizer
OPEN,CLOSE='<ids>','</ids>'
IDCS='⿰⿱⿲⿳⿴⿵⿶⿷⿸⿹⿺⿻'
ARITY={c:3 if c in '⿲⿳' else 2 for c in IDCS}
TOKENS=[OPEN,CLOSE]+list(IDCS)
class IdsEmbedding(nn.Module):
def __init__(self,inner,offset,rows_in):
super().__init__()
count,dims=rows_in.shape
self.inner=inner;self.offset=offset
self.ids_in=rows_in.astype(mx.float32)
self.ids_out=mx.zeros((count,dims),dtype=mx.float32)
self.ids_bias=mx.zeros((count,),dtype=mx.float32)
self.ids_level=mx.zeros((dims,),dtype=mx.float32)
def __call__(self,ids):
new=ids>=self.offset
old=self.inner(mx.where(new,0,ids))
extra=self.ids_in[mx.where(new,ids-self.offset,0)].astype(old.dtype)
return mx.where(new[...,None],extra,old)
def older(self,x):
return self.inner.as_linear(x.astype(self.inner.base.weight.dtype))
def as_linear(self,x):
return mx.concatenate([self.older(x),x.astype(mx.float32)@self.ids_out.T+self.ids_bias],axis=-1)
def level(self,x):
return x.astype(mx.float32)@self.ids_level
def evidence(self,x):
return x.astype(mx.float32)@(self.ids_out[0]-self.ids_level)+self.ids_bias[0]
def ids_tokens(tokenizer,first):
if len(tokenizer)!=first:raise ValueError('Tokenizer size differs from the reader\'s vocabulary')
offset,ids=extend_tokenizer(tokenizer,TOKENS)
if offset!=first:raise ValueError('The IDS tokens do not follow the reader\'s tokens')
return {'first_ids':offset,'open':ids[0],'close':ids[1],'idc':dict(zip(IDCS,ids[2:])),
'arity':{ids[2+i]:ARITY[c] for i,c in enumerate(IDCS)},'vocabulary':offset+len(TOKENS),'tokens':TOKENS}
LAYERS='language_model.model.layers.'
def layer_of(path):
return int(path[len(LAYERS):].split('.',1)[0]) if path.startswith(LAYERS) else None
@contextmanager
def reading_as(model,reader):
held=dict(tree_flatten(model.parameters()));own=[(k,held[k]) for k,_ in reader]
model.load_weights(list(reader),strict=False)
try:yield
finally:model.load_weights(own,strict=False)
def input_rows_from(base,offset,chosen):
"""The input rows of the fourteen IDS tokens, in the order of TOKENS, from the choice kept in ids-tables.json (`inputs`):
<ids> the mean of the "begun" rows (rows of the base vocabulary that the frozen layers know as a character begun and
not finished), at their mean norm; </ids> the row that stands for a finished part; each structure token a begun row of
its own. The input rows were never trained."""
single=[chosen['finished']]+[chosen['structure'][c] for c in IDCS]
if len(set(single))!=len(single) or any(not 0<=t<offset for t in single+chosen['begun']):raise ValueError('The chosen input rows do not fit this model')
row=lambda t:base.weight[t].astype(mx.float32)
begun=base.weight[mx.array(chosen['begun'])].astype(mx.float32);mean=mx.mean(begun,axis=0)
rows={OPEN:mean/mx.linalg.norm(mean)*mx.mean(mx.linalg.norm(begun,axis=-1)),CLOSE:row(chosen['finished']),**{c:row(chosen['structure'][c]) for c in IDCS}}
return mx.stack([rows[t] for t in TOKENS])