"""Single-line inference for the OpenHTR-HS checkpoint.""" from pathlib import Path import numpy as np import torch from PIL import Image from safetensors.torch import load_file from torch import nn from .decoder import OpenHTR_Decoder from .encoder import OpenHTR_Encoder ASSETS = Path(__file__).resolve().parent.parent / "assets" BASE_SHAPES = ((64, 64), (96, 48), (112, 40), (128, 32)) MAX_RATIO = 12 BASE_HEIGHT = 32 MAX_LENGTH = 50 def preprocess_image(image: Image.Image) -> torch.Tensor: """Use the RGB ratio buckets, bicubic resize and normalization of evaluation.""" image = image.convert("RGB") width, height = image.size if height > 1.5 * width: image = image.transpose(Image.Transpose.ROTATE_90) width, height = image.size bucket = min(MAX_RATIO, max(1, round(width / height))) if bucket <= len(BASE_SHAPES): size = BASE_SHAPES[bucket - 1] else: size = (BASE_HEIGHT * bucket, BASE_HEIGHT) resized = image.resize(size, Image.Resampling.BICUBIC) pixels = torch.from_numpy(np.array(resized, dtype=np.uint8)) return pixels.permute(2, 0, 1).contiguous().float().div_(255).sub_(0.5).div_(0.5) class OpenHTR(nn.Module): def __init__(self, vocabulary_size: int): super().__init__() self.encoder = OpenHTR_Encoder( in_channels=3, out_channels=256, dims=[128, 256, 384], depths=[6, 6, 6], num_heads=[4, 8, 12], mixer=[['Conv'] * 6, ['Conv'] * 2 + ['Global'] * 4, ['Global'] * 6], sub_k=[[1, 1], [2, 1], [-1, -1]], last_stage=False, feat2d=False, ) self.decoder = OpenHTR_Decoder( in_channels=self.encoder.out_channels, out_channels=vocabulary_size, nhead=12, num_encoder_layers=-1, num_decoder_layers=2, max_len=MAX_LENGTH, ) def forward(self, pixels: torch.Tensor) -> torch.Tensor: return self.decoder(self.encoder(pixels)) class Recognizer: def __init__(self, device: str | torch.device): self.device = torch.device(device) with (ASSETS / "charset.txt").open("rb") as file: self.characters = tuple(line.decode("utf-8").rstrip("\r\n") for line in file) # The checkpoint orders tokens as EOS, dictionary entries, BOS and PAD. self.model = OpenHTR(len(self.characters) + 3) self.model.load_state_dict(load_file(ASSETS / "openhtr-hs.safetensors"), strict=True) self.model.to(self.device).eval() @torch.inference_mode() def recognize(self, image: Image.Image) -> str: # Composite transparent PNG strokes on paper white before OCR. if "A" in image.getbands() or "transparency" in image.info: image = image.convert("RGBA") paper = Image.new("RGBA", image.size, "white") image = Image.alpha_composite(paper, image) pixels = preprocess_image(image).unsqueeze(0).to(self.device) token_ids = self.model(pixels)[0].argmax(dim=-1).tolist() text = [] for token in token_ids: if token == 0: break text.append(self.characters[token - 1]) return "".join(text)