OpenGlyph-OCR-0 / openglyph_ocr.py
astromandarin's picture
OpenGlyph-OCR-0: the model in two forms (transformers and MLX), model card, licence
853865d verified
Raw History Blame Contribute Delete
13.8 kB
"""OpenGlyph-OCR-0: read one line of printed text that holds rare characters and glyphs without a code point.
(MLX, Apple silicon. Run from a local copy of the repository this file belongs to.)
python openglyph_ocr.py LINE.png [LINE2.png ...]
from openglyph_ocr import load, read_line, atoms
ocr = load() # the folder this file lies in; or load('/path/to/the/folder')
out = read_line(ocr, 'line.png')
print(out['text']) # it looks like: 其字作<gw:u2ff0-u79c0-u6208>
print(atoms(out['text'])) # [('text','其'), ..., ('glyph','u2ff0-u79c0-u6208')]
The picture: one line of horizontal print, as cropped from a page (the lines this model was measured on are about 100 to
130 pixels high and up to 2,800 wide). It is not a page reader: cut the page into lines first.
The text that comes back is ordinary Unicode, except that a glyph without a code point is written <gw:NAME>, NAME being
its GlyphWiki name (https://glyphwiki.org/wiki/NAME): an ebag_s catalog form, another GlyphWiki glyph, an IDS-named glyph
(u2ff0-u79c0-u6208 is ⿰秀戈) or a CDP character.
What is put together here, and why it is done this way. The folder holds the released Xiaomi-OCR-0 weights unchanged
(base/) and, apart from them, every array that training changed (openglyph-ocr-0.safetensors):
LoRA arrays of rank 8 on every linear map of the last six language layers;
the last four vision blocks and the patch merger, in float32;
for each of the 159,052 added tokens (tokens.json) an output row and a bias. Their input rows are all one vector, the
mean input row of some common characters at the usual row length ("a character was written here"), made below from the
base embedding; it was never trained.
Nothing is merged into the base weights: they are 16-bit floats, and folding the arrays in would round them, so the model
would no longer compute what was measured. For the same reason the versions matter; this was measured with
mlx 0.32.3, mlx-vlm 0.3.12, transformers 5.18.0 and tokenizers 0.23.2 (requirements.txt). torch and torchvision are in
that list for the pictures only: with torchvision present transformers prepares them as they were prepared when the model
was measured; without it PIL does, and the pixel values come out up to two grey levels different.
"""
from __future__ import annotations
import hashlib,json,os,sys,time
from pathlib import Path
import mlx.core as mx
import mlx.nn as nn
from mlx.utils import tree_flatten
from tokenizers import AddedToken
HERE=Path(os.path.abspath(__file__)).parent # not resolved: in a hub snapshot this file is a link into a store of blobs
PROMPT=('Transcribe the image in reading order. Output ordinary characters as Unicode text. '
'For a catalog-specific historical glyph, output <gw:GLYPH_ID>, replacing GLYPH_ID with its exact ebag_s catalog name. '
'Read vertical columns top-to-bottom and right-to-left. Preserve punctuation and line breaks. '
'Return only the transcription, without markdown fences or explanation.') # the prompt of training, word for word
LORA={'rank':8,'alpha':1.0,'dropout':0.05,'last_layers':6} # as trained; the dropout does nothing when reading
VISION_TAIL=4 # the last vision blocks that were trained (with the patch merger)
SAMPLE='之不以而其為於者也有人此所文字曰天下' # the common characters whose mean input row every added token has
TRAINED='openglyph-ocr-0.safetensors'
ROWS='language_model.model.embed_tokens.extra_out';BIAS='language_model.model.embed_tokens.extra_bias'
BASE_FILES=('processor_config.json','tokenizer_config.json','chat_template.jinja','tokenizer.json','model.safetensors')
class ExtendedEmbedding(nn.Module):
"""Ids below `offset` use the base matrix; the added tokens own separate input rows, output rows and biases.
Token id i owns logit column i."""
def __init__(self,base,offset,count):
super().__init__()
if offset>base.weight.shape[0]:raise ValueError('Tokenizer is larger than the base embedding')
self.base=base;self.offset=offset;dims=base.weight.shape[1]
self.extra_in=mx.zeros((count,dims),dtype=mx.float32)
self.extra_out=mx.zeros((count,dims),dtype=mx.float32)
self.extra_bias=mx.zeros((count,),dtype=mx.float32)
def __call__(self,ids):
new=ids>=self.offset
base=self.base(mx.where(new,0,ids))
extra=self.extra_in[mx.where(new,ids-self.offset,0)].astype(base.dtype)
return mx.where(new[...,None],extra,base)
def as_linear(self,x):
base=self.base.as_linear(x)[...,:self.offset].astype(mx.float32)
return mx.concatenate([base,x.astype(mx.float32)@self.extra_out.T+self.extra_bias],axis=-1)
class OCR:
def __init__(self,model,processor,offset,folder):self.model=model;self.processor=processor;self.offset=offset;self.folder=folder
def runtime_view(base,cache=None):
"""A folder of links to the base files with one word of config.json changed: the released config names the vision
tower 'qwen3_5_vision', which mlx-vlm 0.3.12 knows as 'qwen3_5'. No weight is touched and base/ is left as it is."""
base=Path(os.path.abspath(base));config=json.loads((base/'config.json').read_text(encoding='utf-8'))
if config.get('vision_config',{}).get('model_type')=='qwen3_5_vision':config['vision_config']['model_type']='qwen3_5'
text=json.dumps(config,ensure_ascii=False,indent=2)+'\n'
key=hashlib.sha256((str(base)+'\0'+text).encode('utf-8')).hexdigest()[:16]
view=Path(cache or os.environ.get('OPENGLYPH_OCR_CACHE') or Path.home()/'.cache'/'openglyph-ocr')/('view-'+key)
view.mkdir(parents=True,exist_ok=True);(view/'config.json').write_text(text,encoding='utf-8')
for name in BASE_FILES:
source=base/name;target=view/name
if not source.exists():raise FileNotFoundError(str(source))
if target.is_symlink() and os.readlink(target)==str(source):continue
if target.is_symlink() or target.exists():target.unlink()
target.symlink_to(source)
return view
def added_tokens(folder):
table=json.loads((Path(folder)/'tokens.json').read_text(encoding='utf-8'))
return table['catalog_forms']+table['extension_characters']+table['glyphwiki_glyphs'],table['first_id']
def load(folder=None,cache=None):
"""Build the model as it was trained and measured, from the folder that holds base/, the trained arrays and the
tokens. Takes some seconds."""
from mlx_vlm.utils import load as load_model
from mlx_vlm.trainer.lora import LoRaLayer
from mlx_vlm.trainer.utils import set_module_by_name
folder=Path(os.path.abspath(folder)) if folder else HERE
model,processor=load_model(str(runtime_view(folder/'base',cache)),processor_config={'trust_remote_code':False,'local_files_only':True})
if model.config.model_type!='qwen3_5':raise ValueError('Not the Qwen3.5 architecture of Xiaomi-OCR-0')
pictures_by=getattr(getattr(processor,'image_processor',None),'backend',None)
if pictures_by is not None and pictures_by!='torchvision': # without torchvision, transformers prepares the pictures with PIL
import warnings
warnings.warn(f'The pictures are prepared with {pictures_by}, not with torchvision as they were when this model was measured: '
'the pixel values differ by up to two grey levels. Install torch and torchvision (requirements.txt).')
tokenizer=processor.tokenizer
backend=getattr(tokenizer,'backend_tokenizer',tokenizer);backend.normalizer=None # the characters of the source are kept as they are
if backend.normalizer is not None:raise ValueError('Could not switch off the normalisation of the tokenizer')
# LoRA on every linear map of the last six language layers
model.freeze();depth=len(model.language_model.model.layers)
for name,module in list(model.language_model.named_modules()):
parts=name.split('.')
if not isinstance(module,nn.Linear) or 'layers' not in parts:continue
if int(parts[parts.index('layers')+1])<depth-LORA['last_layers']:continue
set_module_by_name(model.language_model,name,LoRaLayer(module,LORA['rank'],LORA['alpha'],LORA['dropout']))
# the added tokens: fresh ids after the base vocabulary, in the order of tokens.json
tokens,first=added_tokens(folder);offset=len(tokenizer)
added=tokenizer.add_tokens([AddedToken(t,single_word=False,lstrip=False,rstrip=False,normalized=False,special=False) for t in tokens])
if offset!=first or added!=len(tokens) or tokenizer.convert_tokens_to_ids(list(tokens))!=list(range(offset,offset+len(tokens))):
raise ValueError('The added tokens did not get the ids they were trained with (another version of the tokenizer?)')
lm=model.language_model
if not lm.args.tie_word_embeddings:raise ValueError('Expected tied input and output embeddings')
lm.model.embed_tokens=ExtendedEmbedding(lm.model.embed_tokens,offset,len(tokens));emb=lm.model.embed_tokens
# their input rows: one generic vector
ids=[tokenizer.encode(c,add_special_tokens=False) for c in SAMPLE]
if any(len(i)!=1 for i in ids):raise ValueError('A sample character is not a single base token')
rows=emb.base.weight[mx.array([i[0] for i in ids])].astype(mx.float32)
mean=mx.mean(rows,axis=0);norm=mx.mean(mx.linalg.norm(rows,axis=-1))
emb.extra_in=mx.broadcast_to((mean/mx.linalg.norm(mean)*norm)[None,:],emb.extra_in.shape)+0
# the trained arrays; every LoRA array and every array of the vision tail must come from the file
weights=mx.load(str(folder/TRAINED));tower=model.vision_tower;blocks=len(tower.blocks)
lora={k for k,_ in tree_flatten(model.trainable_parameters()) if k.endswith(('.A','.B'))}
tail={f'vision_tower.blocks.{i}.{k}' for i in range(blocks-VISION_TAIL,blocks) for k,_ in tree_flatten(tower.blocks[i].parameters())}
tail|={'vision_tower.merger.'+k for k,_ in tree_flatten(tower.merger.parameters())}
if set(weights)!=lora|tail|{ROWS,BIAS}:raise ValueError('The trained arrays are not those this model takes (another version of mlx-vlm?)')
if tuple(weights[ROWS].shape)!=tuple(emb.extra_out.shape) or tuple(weights[BIAS].shape)!=tuple(emb.extra_bias.shape):raise ValueError('The output rows are not as many as the added tokens')
model.load_weights(list(weights.items()),strict=False);mx.eval(model.parameters());model.eval()
return OCR(model,processor,offset,folder)
def decode_ids(tokenizer,tokens,offset):
"""The whole sequence decoded as UTF-8 at once; an added token gives its own text."""
from mlx_vlm.tokenizer_utils import BPEStreamingDetokenizer
BPEStreamingDetokenizer.make_byte_decoder() # the table from the characters of a BPE piece to bytes; filled once, and not left to whoever built a detokenizer before
pieces=tokenizer.convert_ids_to_tokens(tokens);mapping=BPEStreamingDetokenizer._byte_decoder;raw=bytearray()
for token,piece in zip(tokens,pieces):
raw+=piece.encode('utf-8') if token>=offset else bytes(mapping[c] for c in piece)
return raw.decode('utf-8',errors='strict')
def read_line(ocr,image,max_tokens=256):
"""Read one line picture (a path). Greedy, nothing constrained. Returns the text, the token ids, whether the model
ended the line itself (`complete`; False: it ran into max_tokens) and the seconds it took."""
from mlx_vlm.utils import prepare_inputs
from mlx_vlm.generate import generate_step,wired_limit,generation_stream
model,processor=ocr.model,ocr.processor;model.eval();start=time.monotonic()
prompt=processor.apply_chat_template([{'role':'user','content':[{'type':'image'},{'type':'text','text':PROMPT}]}],
tokenize=False,add_generation_prompt=True,enable_thinking=False)
values=prepare_inputs(processor,prompts=[prompt],images=str(image),image_token_index=model.config.image_token_id,add_special_tokens=False)
extras={k:v for k,v in values.items() if k not in ('input_ids','pixel_values','attention_mask')}
tokens=[];ended=False
with wired_limit(model,[generation_stream]):
iterator=generate_step(values['input_ids'],model,values.get('pixel_values'),values.get('attention_mask'),
max_tokens=max_tokens,temperature=0.,logits_processors=[],**extras)
try:
for token,_ in iterator:
if processor.tokenizer.stopping_criteria(token):ended=True;break
tokens.append(token)
finally:iterator.close()
tokenizer=processor.tokenizer;valid=True
try:text=decode_ids(tokenizer,tokens,ocr.offset)
except (UnicodeDecodeError,KeyError,TypeError): # bytes that are no UTF-8: kept as the tokenizer shows them, and said
valid=False;text=tokenizer.decode(tokens,skip_special_tokens=False,clean_up_tokenization_spaces=False)
mx.clear_cache()
return {'text':text,'token_ids':tokens,'complete':ended,'valid_utf8':valid,'added_tokens':sum(t>=ocr.offset for t in tokens),'seconds':time.monotonic()-start}
def atoms(text):
"""A reading as a list of ('text', character) and ('glyph', GlyphWiki name)."""
out=[];i=0
while i<len(text):
end=text.find('>',i+4) if text.startswith('<gw:',i) else -1
if end>0:out.append(('glyph',text[i+4:end]));i=end+1
else:out.append(('text',text[i]));i+=1
return out
def main():
if len(sys.argv)<2:raise SystemExit(__doc__)
ocr=load()
for path in sys.argv[1:]:
out=read_line(ocr,path)
print(json.dumps({'image':path,'text':out['text'],'complete':out['complete'],'seconds':round(out['seconds'],2)},ensure_ascii=False),flush=True)
if __name__=='__main__':main()