OpenGlyph-OCR-0 / modeling_openglyph_ocr.py
astromandarin's picture
OpenGlyph-OCR-0: the model in two forms (transformers and MLX), model card, licence
853865d verified
Raw History Blame Contribute Delete
4.45 kB
"""OpenGlyph-OCR-0 for PyTorch: Qwen3.5 (Xiaomi-OCR-0) whose output layer has a bias and is not tied to the input embedding.
import torch
from transformers import AutoModelForImageTextToText, AutoProcessor
repo = "OpenGlyph/OpenGlyph-OCR-0"
model = AutoModelForImageTextToText.from_pretrained(repo, trust_remote_code=True, dtype=torch.float32).eval()
processor = AutoProcessor.from_pretrained(repo, trust_remote_code=True)
print(model.read_line(processor, "line.png")["text"])
Everything but the output layer is the stock model. The 159,052 added tokens (every CJK extension character and the
GlyphWiki glyphs without a code point, written <gw:NAME>) each have an output row and a bias of their own; their input
rows are one and the same vector. The weights that training changed are stored in float32 and the rest as the base
model's 16-bit floats, so load the model in float32 (`dtype=torch.float32`, which is also what its config asks for): every
weight is then exactly the trained one. Loaded in 16-bit floats the trained weights are rounded, and that was not
measured.
"""
import warnings
import torch
from torch import nn
from transformers.models.qwen3_5.modeling_qwen3_5 import (
Qwen3_5ForConditionalGeneration,
Qwen3_5Model,
Qwen3_5PreTrainedModel,
)
from .configuration_openglyph_ocr import OpenGlyphOCRConfig
# The prompt of training, word for word.
PROMPT = (
"Transcribe the image in reading order. Output ordinary characters as Unicode text. "
"For a catalog-specific historical glyph, output <gw:GLYPH_ID>, replacing GLYPH_ID with its exact ebag_s catalog name. "
"Read vertical columns top-to-bottom and right-to-left. Preserve punctuation and line breaks. "
"Return only the transcription, without markdown fences or explanation."
)
class OpenGlyphOCRForConditionalGeneration(Qwen3_5ForConditionalGeneration):
config_class = OpenGlyphOCRConfig
_tied_weights_keys = {} # the head is not tied: an added token's output row is not its input row
def __init__(self, config):
Qwen3_5PreTrainedModel.__init__(self, config)
self.model = Qwen3_5Model(config)
# as the stock class, but with a bias: nought for the base vocabulary, trained for every added token
self.lm_head = nn.Linear(config.text_config.hidden_size, config.text_config.vocab_size, bias=True)
self.post_init()
@torch.no_grad()
def read_line(self, processor, image, max_new_tokens=256):
"""Read one line picture (a path or a PIL image): greedy, nothing constrained.
Returns the text, the token ids, and whether the model ended the line itself (`complete`; False: it ran into
max_new_tokens). A glyph without a code point is written <gw:NAME>, NAME being its GlyphWiki name."""
from PIL import Image
if not isinstance(image, Image.Image):
image = Image.open(image)
image = image.convert("RGB")
backend = getattr(getattr(processor, "image_processor", None), "backend", None)
if backend is not None and backend != "torchvision":
# without torchvision, transformers prepares the picture with PIL
warnings.warn(
f"The picture is prepared with {backend}, not with torchvision as the pictures were when this model "
"was measured: the pixel values differ by up to two grey levels. Install torchvision."
)
messages =[{"role": "user", "content": [{"type": "image"}, {"type": "text", "text": PROMPT}]}]
prompt = processor.apply_chat_template(
messages, tokenize=False, add_generation_prompt=True, enable_thinking=False
)
inputs = processor(text=[prompt], images=[image], padding=True, return_tensors="pt")
inputs = inputs.to(self.device)
output = self.generate(**inputs, max_new_tokens=max_new_tokens, do_sample=False)
tokens = output[0, inputs["input_ids"].shape[1] :].tolist()
stop = self.generation_config.eos_token_id
stop = set(stop if isinstance(stop, (list, tuple)) else [stop])
complete = bool(tokens) and tokens[-1] in stop
if complete:
tokens = tokens[:-1]
text = processor.tokenizer.decode(tokens, skip_special_tokens=False, clean_up_tokenization_spaces=False)
return {"text": text, "token_ids": tokens, "complete": complete}
__all__ = ["OpenGlyphOCRForConditionalGeneration"]