Image-Text-to-Text
MLX
Safetensors
Chinese
ocr
rare-characters
ids
ideographic-description-sequence
glyphwiki
palaeography
Instructions to use OpenGlyph/OpenGlyph-OCR-IDS with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- MLX
How to use OpenGlyph/OpenGlyph-OCR-IDS with MLX:
# Make sure mlx-vlm is installed # pip install --upgrade mlx-vlm from mlx_vlm import load, generate from mlx_vlm.prompt_utils import apply_chat_template from mlx_vlm.utils import load_config # Load the model model, processor = load("OpenGlyph/OpenGlyph-OCR-IDS") config = load_config("OpenGlyph/OpenGlyph-OCR-IDS") # Prepare input image = ["http://images.cocodataset.org/val2017/000000039769.jpg"] prompt = "Describe this image." # Apply chat template formatted_prompt = apply_chat_template( processor, config, prompt, num_images=1 ) # Generate output output = generate(model, processor, formatted_prompt, image) print(output) - Notebooks
- Google Colab
- Kaggle
- Local Apps Settings
- LM Studio
- Atomic Chat
Download openglyph_ocr_ids/model.py from OpenGlyph/OpenGlyph-OCR-IDS: direct link, hf CLI and curl.
- Browser
- Download file 4.07 kB
-
https://huggingface.co/OpenGlyph/OpenGlyph-OCR-IDS/resolve/main/openglyph_ocr_ids/model.py
- Command line
-
hf download hf://OpenGlyph/OpenGlyph-OCR-IDS/openglyph_ocr_ids/model.py
-
curl -L -o model.py https://huggingface.co/OpenGlyph/OpenGlyph-OCR-IDS/resolve/main/openglyph_ocr_ids/model.py
4.07 kB
| """The IDS layer on top of OpenGlyph-OCR-0: fourteen added tokens (<ids>, </ids> and the twelve structure characters | |
| ⿰⿱⿲⿳⿴⿵⿶⿷⿸⿹⿺⿻) with fixed input rows -- rows the frozen layers know as "a character was begun and is not finished" -- | |
| and output rows and biases of their own; and the swapping of one set of weights for another for a while (reading_as). | |
| Four sets of weights work in the one model: the reader (OpenGlyph-OCR-0's own, never changed), which writes the main | |
| line; the opener, which says where an IDS begins; the part reader, which reads the cells of an unfolded glyph; the | |
| speller, which spells an IDS from the glyph as it stands. The model carries the opener's weights; the others are put in | |
| for a while and taken out again.""" | |
| from __future__ import annotations | |
| from contextlib import contextmanager | |
| import mlx.core as mx | |
| import mlx.nn as nn | |
| from mlx.utils import tree_flatten | |
| from .vocab import extend_tokenizer | |
| OPEN,CLOSE='<ids>','</ids>' | |
| IDCS='⿰⿱⿲⿳⿴⿵⿶⿷⿸⿹⿺⿻' | |
| ARITY={c:3 if c in '⿲⿳' else 2 for c in IDCS} | |
| TOKENS=[OPEN,CLOSE]+list(IDCS) | |
| class IdsEmbedding(nn.Module): | |
| def __init__(self,inner,offset,rows_in): | |
| super().__init__() | |
| count,dims=rows_in.shape | |
| self.inner=inner;self.offset=offset | |
| self.ids_in=rows_in.astype(mx.float32) | |
| self.ids_out=mx.zeros((count,dims),dtype=mx.float32) | |
| self.ids_bias=mx.zeros((count,),dtype=mx.float32) | |
| self.ids_level=mx.zeros((dims,),dtype=mx.float32) | |
| def __call__(self,ids): | |
| new=ids>=self.offset | |
| old=self.inner(mx.where(new,0,ids)) | |
| extra=self.ids_in[mx.where(new,ids-self.offset,0)].astype(old.dtype) | |
| return mx.where(new[...,None],extra,old) | |
| def older(self,x): | |
| return self.inner.as_linear(x.astype(self.inner.base.weight.dtype)) | |
| def as_linear(self,x): | |
| return mx.concatenate([self.older(x),x.astype(mx.float32)@self.ids_out.T+self.ids_bias],axis=-1) | |
| def level(self,x): | |
| return x.astype(mx.float32)@self.ids_level | |
| def evidence(self,x): | |
| return x.astype(mx.float32)@(self.ids_out[0]-self.ids_level)+self.ids_bias[0] | |
| def ids_tokens(tokenizer,first): | |
| if len(tokenizer)!=first:raise ValueError('Tokenizer size differs from the reader\'s vocabulary') | |
| offset,ids=extend_tokenizer(tokenizer,TOKENS) | |
| if offset!=first:raise ValueError('The IDS tokens do not follow the reader\'s tokens') | |
| return {'first_ids':offset,'open':ids[0],'close':ids[1],'idc':dict(zip(IDCS,ids[2:])), | |
| 'arity':{ids[2+i]:ARITY[c] for i,c in enumerate(IDCS)},'vocabulary':offset+len(TOKENS),'tokens':TOKENS} | |
| LAYERS='language_model.model.layers.' | |
| def layer_of(path): | |
| return int(path[len(LAYERS):].split('.',1)[0]) if path.startswith(LAYERS) else None | |
| def reading_as(model,reader): | |
| held=dict(tree_flatten(model.parameters()));own=[(k,held[k]) for k,_ in reader] | |
| model.load_weights(list(reader),strict=False) | |
| try:yield | |
| finally:model.load_weights(own,strict=False) | |
| def input_rows_from(base,offset,chosen): | |
| """The input rows of the fourteen IDS tokens, in the order of TOKENS, from the choice kept in ids-tables.json (`inputs`): | |
| <ids> the mean of the "begun" rows (rows of the base vocabulary that the frozen layers know as a character begun and | |
| not finished), at their mean norm; </ids> the row that stands for a finished part; each structure token a begun row of | |
| its own. The input rows were never trained.""" | |
| single=[chosen['finished']]+[chosen['structure'][c] for c in IDCS] | |
| if len(set(single))!=len(single) or any(not 0<=t<offset for t in single+chosen['begun']):raise ValueError('The chosen input rows do not fit this model') | |
| row=lambda t:base.weight[t].astype(mx.float32) | |
| begun=base.weight[mx.array(chosen['begun'])].astype(mx.float32);mean=mx.mean(begun,axis=0) | |
| rows={OPEN:mean/mx.linalg.norm(mean)*mx.mean(mx.linalg.norm(begun,axis=-1)),CLOSE:row(chosen['finished']),**{c:row(chosen['structure'][c]) for c in IDCS}} | |
| return mx.stack([rows[t] for t in TOKENS]) | |