"""OpenGlyph-OCR-0: read one line of printed text that holds rare characters and glyphs without a code point. (MLX, Apple silicon. Run from a local copy of the repository this file belongs to.) python openglyph_ocr.py LINE.png [LINE2.png ...] from openglyph_ocr import load, read_line, atoms ocr = load() # the folder this file lies in; or load('/path/to/the/folder') out = read_line(ocr, 'line.png') print(out['text']) # it looks like: 其字作 print(atoms(out['text'])) # [('text','其'), ..., ('glyph','u2ff0-u79c0-u6208')] The picture: one line of horizontal print, as cropped from a page (the lines this model was measured on are about 100 to 130 pixels high and up to 2,800 wide). It is not a page reader: cut the page into lines first. The text that comes back is ordinary Unicode, except that a glyph without a code point is written , NAME being its GlyphWiki name (https://glyphwiki.org/wiki/NAME): an ebag_s catalog form, another GlyphWiki glyph, an IDS-named glyph (u2ff0-u79c0-u6208 is ⿰秀戈) or a CDP character. What is put together here, and why it is done this way. The folder holds the released Xiaomi-OCR-0 weights unchanged (base/) and, apart from them, every array that training changed (openglyph-ocr-0.safetensors): LoRA arrays of rank 8 on every linear map of the last six language layers; the last four vision blocks and the patch merger, in float32; for each of the 159,052 added tokens (tokens.json) an output row and a bias. Their input rows are all one vector, the mean input row of some common characters at the usual row length ("a character was written here"), made below from the base embedding; it was never trained. Nothing is merged into the base weights: they are 16-bit floats, and folding the arrays in would round them, so the model would no longer compute what was measured. For the same reason the versions matter; this was measured with mlx 0.32.3, mlx-vlm 0.3.12, transformers 5.18.0 and tokenizers 0.23.2 (requirements.txt). torch and torchvision are in that list for the pictures only: with torchvision present transformers prepares them as they were prepared when the model was measured; without it PIL does, and the pixel values come out up to two grey levels different. """ from __future__ import annotations import hashlib,json,os,sys,time from pathlib import Path import mlx.core as mx import mlx.nn as nn from mlx.utils import tree_flatten from tokenizers import AddedToken HERE=Path(os.path.abspath(__file__)).parent # not resolved: in a hub snapshot this file is a link into a store of blobs PROMPT=('Transcribe the image in reading order. Output ordinary characters as Unicode text. ' 'For a catalog-specific historical glyph, output , replacing GLYPH_ID with its exact ebag_s catalog name. ' 'Read vertical columns top-to-bottom and right-to-left. Preserve punctuation and line breaks. ' 'Return only the transcription, without markdown fences or explanation.') # the prompt of training, word for word LORA={'rank':8,'alpha':1.0,'dropout':0.05,'last_layers':6} # as trained; the dropout does nothing when reading VISION_TAIL=4 # the last vision blocks that were trained (with the patch merger) SAMPLE='之不以而其為於者也有人此所文字曰天下' # the common characters whose mean input row every added token has TRAINED='openglyph-ocr-0.safetensors' ROWS='language_model.model.embed_tokens.extra_out';BIAS='language_model.model.embed_tokens.extra_bias' BASE_FILES=('processor_config.json','tokenizer_config.json','chat_template.jinja','tokenizer.json','model.safetensors') class ExtendedEmbedding(nn.Module): """Ids below `offset` use the base matrix; the added tokens own separate input rows, output rows and biases. Token id i owns logit column i.""" def __init__(self,base,offset,count): super().__init__() if offset>base.weight.shape[0]:raise ValueError('Tokenizer is larger than the base embedding') self.base=base;self.offset=offset;dims=base.weight.shape[1] self.extra_in=mx.zeros((count,dims),dtype=mx.float32) self.extra_out=mx.zeros((count,dims),dtype=mx.float32) self.extra_bias=mx.zeros((count,),dtype=mx.float32) def __call__(self,ids): new=ids>=self.offset base=self.base(mx.where(new,0,ids)) extra=self.extra_in[mx.where(new,ids-self.offset,0)].astype(base.dtype) return mx.where(new[...,None],extra,base) def as_linear(self,x): base=self.base.as_linear(x)[...,:self.offset].astype(mx.float32) return mx.concatenate([base,x.astype(mx.float32)@self.extra_out.T+self.extra_bias],axis=-1) class OCR: def __init__(self,model,processor,offset,folder):self.model=model;self.processor=processor;self.offset=offset;self.folder=folder def runtime_view(base,cache=None): """A folder of links to the base files with one word of config.json changed: the released config names the vision tower 'qwen3_5_vision', which mlx-vlm 0.3.12 knows as 'qwen3_5'. No weight is touched and base/ is left as it is.""" base=Path(os.path.abspath(base));config=json.loads((base/'config.json').read_text(encoding='utf-8')) if config.get('vision_config',{}).get('model_type')=='qwen3_5_vision':config['vision_config']['model_type']='qwen3_5' text=json.dumps(config,ensure_ascii=False,indent=2)+'\n' key=hashlib.sha256((str(base)+'\0'+text).encode('utf-8')).hexdigest()[:16] view=Path(cache or os.environ.get('OPENGLYPH_OCR_CACHE') or Path.home()/'.cache'/'openglyph-ocr')/('view-'+key) view.mkdir(parents=True,exist_ok=True);(view/'config.json').write_text(text,encoding='utf-8') for name in BASE_FILES: source=base/name;target=view/name if not source.exists():raise FileNotFoundError(str(source)) if target.is_symlink() and os.readlink(target)==str(source):continue if target.is_symlink() or target.exists():target.unlink() target.symlink_to(source) return view def added_tokens(folder): table=json.loads((Path(folder)/'tokens.json').read_text(encoding='utf-8')) return table['catalog_forms']+table['extension_characters']+table['glyphwiki_glyphs'],table['first_id'] def load(folder=None,cache=None): """Build the model as it was trained and measured, from the folder that holds base/, the trained arrays and the tokens. Takes some seconds.""" from mlx_vlm.utils import load as load_model from mlx_vlm.trainer.lora import LoRaLayer from mlx_vlm.trainer.utils import set_module_by_name folder=Path(os.path.abspath(folder)) if folder else HERE model,processor=load_model(str(runtime_view(folder/'base',cache)),processor_config={'trust_remote_code':False,'local_files_only':True}) if model.config.model_type!='qwen3_5':raise ValueError('Not the Qwen3.5 architecture of Xiaomi-OCR-0') pictures_by=getattr(getattr(processor,'image_processor',None),'backend',None) if pictures_by is not None and pictures_by!='torchvision': # without torchvision, transformers prepares the pictures with PIL import warnings warnings.warn(f'The pictures are prepared with {pictures_by}, not with torchvision as they were when this model was measured: ' 'the pixel values differ by up to two grey levels. Install torch and torchvision (requirements.txt).') tokenizer=processor.tokenizer backend=getattr(tokenizer,'backend_tokenizer',tokenizer);backend.normalizer=None # the characters of the source are kept as they are if backend.normalizer is not None:raise ValueError('Could not switch off the normalisation of the tokenizer') # LoRA on every linear map of the last six language layers model.freeze();depth=len(model.language_model.model.layers) for name,module in list(model.language_model.named_modules()): parts=name.split('.') if not isinstance(module,nn.Linear) or 'layers' not in parts:continue if int(parts[parts.index('layers')+1])=offset else bytes(mapping[c] for c in piece) return raw.decode('utf-8',errors='strict') def read_line(ocr,image,max_tokens=256): """Read one line picture (a path). Greedy, nothing constrained. Returns the text, the token ids, whether the model ended the line itself (`complete`; False: it ran into max_tokens) and the seconds it took.""" from mlx_vlm.utils import prepare_inputs from mlx_vlm.generate import generate_step,wired_limit,generation_stream model,processor=ocr.model,ocr.processor;model.eval();start=time.monotonic() prompt=processor.apply_chat_template([{'role':'user','content':[{'type':'image'},{'type':'text','text':PROMPT}]}], tokenize=False,add_generation_prompt=True,enable_thinking=False) values=prepare_inputs(processor,prompts=[prompt],images=str(image),image_token_index=model.config.image_token_id,add_special_tokens=False) extras={k:v for k,v in values.items() if k not in ('input_ids','pixel_values','attention_mask')} tokens=[];ended=False with wired_limit(model,[generation_stream]): iterator=generate_step(values['input_ids'],model,values.get('pixel_values'),values.get('attention_mask'), max_tokens=max_tokens,temperature=0.,logits_processors=[],**extras) try: for token,_ in iterator: if processor.tokenizer.stopping_criteria(token):ended=True;break tokens.append(token) finally:iterator.close() tokenizer=processor.tokenizer;valid=True try:text=decode_ids(tokenizer,tokens,ocr.offset) except (UnicodeDecodeError,KeyError,TypeError): # bytes that are no UTF-8: kept as the tokenizer shows them, and said valid=False;text=tokenizer.decode(tokens,skip_special_tokens=False,clean_up_tokenization_spaces=False) mx.clear_cache() return {'text':text,'token_ids':tokens,'complete':ended,'valid_utf8':valid,'added_tokens':sum(t>=ocr.offset for t in tokens),'seconds':time.monotonic()-start} def atoms(text): """A reading as a list of ('text', character) and ('glyph', GlyphWiki name).""" out=[];i=0 while i',i+4) if text.startswith('0:out.append(('glyph',text[i+4:end]));i=end+1 else:out.append(('text',text[i]));i+=1 return out def main(): if len(sys.argv)<2:raise SystemExit(__doc__) ocr=load() for path in sys.argv[1:]: out=read_line(ocr,path) print(json.dumps({'image':path,'text':out['text'],'complete':out['complete'],'seconds':round(out['seconds'],2)},ensure_ascii=False),flush=True) if __name__=='__main__':main()