OpenHTR-Rec / openhtr /inference.py
OpenHTR's picture
Upload 12 files
b417724 verified
Raw History Blame Contribute Delete
3.28 kB
"""Single-line inference for the OpenHTR-HS checkpoint."""
from pathlib import Path
import numpy as np
import torch
from PIL import Image
from safetensors.torch import load_file
from torch import nn
from .decoder import OpenHTR_Decoder
from .encoder import OpenHTR_Encoder
ASSETS = Path(__file__).resolve().parent.parent / "assets"
BASE_SHAPES = ((64, 64), (96, 48), (112, 40), (128, 32))
MAX_RATIO = 12
BASE_HEIGHT = 32
MAX_LENGTH = 50
def preprocess_image(image: Image.Image) -> torch.Tensor:
"""Use the RGB ratio buckets, bicubic resize and normalization of evaluation."""
image = image.convert("RGB")
width, height = image.size
if height > 1.5 * width:
image = image.transpose(Image.Transpose.ROTATE_90)
width, height = image.size
bucket = min(MAX_RATIO, max(1, round(width / height)))
if bucket <= len(BASE_SHAPES):
size = BASE_SHAPES[bucket - 1]
else:
size = (BASE_HEIGHT * bucket, BASE_HEIGHT)
resized = image.resize(size, Image.Resampling.BICUBIC)
pixels = torch.from_numpy(np.array(resized, dtype=np.uint8))
return pixels.permute(2, 0, 1).contiguous().float().div_(255).sub_(0.5).div_(0.5)
class OpenHTR(nn.Module):
def __init__(self, vocabulary_size: int):
super().__init__()
self.encoder = OpenHTR_Encoder(
in_channels=3,
out_channels=256,
dims=[128, 256, 384],
depths=[6, 6, 6],
num_heads=[4, 8, 12],
mixer=[['Conv'] * 6, ['Conv'] * 2 + ['Global'] * 4, ['Global'] * 6],
sub_k=[[1, 1], [2, 1], [-1, -1]],
last_stage=False,
feat2d=False,
)
self.decoder = OpenHTR_Decoder(
in_channels=self.encoder.out_channels,
out_channels=vocabulary_size,
nhead=12,
num_encoder_layers=-1,
num_decoder_layers=2,
max_len=MAX_LENGTH,
)
def forward(self, pixels: torch.Tensor) -> torch.Tensor:
return self.decoder(self.encoder(pixels))
class Recognizer:
def __init__(self, device: str | torch.device):
self.device = torch.device(device)
with (ASSETS / "charset.txt").open("rb") as file:
self.characters = tuple(line.decode("utf-8").rstrip("\r\n") for line in file)
# The checkpoint orders tokens as EOS, dictionary entries, BOS and PAD.
self.model = OpenHTR(len(self.characters) + 3)
self.model.load_state_dict(load_file(ASSETS / "openhtr-hs.safetensors"), strict=True)
self.model.to(self.device).eval()
@torch.inference_mode()
def recognize(self, image: Image.Image) -> str:
# Composite transparent PNG strokes on paper white before OCR.
if "A" in image.getbands() or "transparency" in image.info:
image = image.convert("RGBA")
paper = Image.new("RGBA", image.size, "white")
image = Image.alpha_composite(paper, image)
pixels = preprocess_image(image).unsqueeze(0).to(self.device)
token_ids = self.model(pixels)[0].argmax(dim=-1).tolist()
text = []
for token in token_ids:
if token == 0:
break
text.append(self.characters[token - 1])
return "".join(text)