openocrv2 / modeling_openocrv2_det.py
Yesianrohn's picture
Update modeling_openocrv2_det.py
0e38cbe verified
Raw History Blame Contribute Delete
35.7 kB
import numpy as np
import torch
from torch import nn
from torch.nn import functional as F
from transformers import PretrainedConfig
from transformers import PreTrainedModel
from transformers.modeling_outputs import ModelOutput
from PIL import Image
from dataclasses import dataclass
class DCNv2(nn.Module):
"""Modulated deformable 3x3 convolution with learned offsets and masks.
Parameter names match the frozen checkpoint (``weight``, ``conv_offset.*``).
``forward`` dispatches to the portable path so this class is importable
without torchvision.
"""
def __init__(self, channels, stride=1, deform_groups=2):
super().__init__()
self.in_channels = channels
self.out_channels = channels
self.stride = (stride, stride)
self.padding = (1, 1)
self.dilation = (1, 1)
self.deform_groups = deform_groups
# Registered the same way nn.Conv2d would, so key names line up.
self.weight = nn.Parameter(torch.empty(channels, channels, 3, 3))
self.bias = None
self.conv_offset = nn.Conv2d(
channels, deform_groups * 27, 3, stride=stride, padding=1
)
def forward(self, x):
offset_y, offset_x, mask = self.conv_offset(x).chunk(3, dim=1)
return deform_conv2d_portable(
x,
torch.cat((offset_y, offset_x), dim=1),
self.weight,
self.bias,
self.stride,
self.padding,
self.dilation,
mask=mask.sigmoid(),
groups=self.deform_groups,
)
def deform_conv2d_portable(
x,
offset,
weight,
bias=None,
stride=(1, 1),
padding=(1, 1),
dilation=(1, 1),
mask=None,
groups=1,
):
"""Modulated deformable convolution via ``grid_sample``.
``offset`` has ``2 * groups * 9`` channels ordered as
``[group][ky][kx][dy, dx]`` — the layout ``torchvision`` expects and the one
the trained ``conv_offset`` produces.
"""
if isinstance(stride, int):
stride = (stride, stride)
if isinstance(padding, int):
padding = (padding, padding)
if isinstance(dilation, int):
dilation = (dilation, dilation)
if (stride, padding, dilation) != ((1, 1), (1, 1), (1, 1)):
raise ValueError(
"Portable DCN covers the frozen stride-1, pad-1, dilation-1 adapters only"
)
batch, channels, height, width = x.shape
if offset.shape[1] != groups * 18:
raise ValueError(
f"offset has {offset.shape[1]} channels; expected {groups * 18} "
f"for {groups} deform groups"
)
dtype = x.dtype
device = x.device
delta = offset.reshape(batch, groups, 9, 2, height, width)
yy, xx = torch.meshgrid(
torch.arange(height, device=device, dtype=torch.float32),
torch.arange(width, device=device, dtype=torch.float32),
indexing="ij",
)
ky, kx = torch.meshgrid(
torch.arange(3, device=device, dtype=torch.float32),
torch.arange(3, device=device, dtype=torch.float32),
indexing="ij",
)
# Coordinates are expressed in the *unpadded* input frame; padding is handled
# by ``padding_mode="zeros"``. Adding a padding offset here would shift every
# sample by ``padding`` pixels (verified against torchvision: it does not).
y = yy[None, None, None] + (ky.reshape(1, 1, 9, 1, 1) - 1) * dilation[0] + delta[:, :, :, 0]
xcoord = (
xx[None, None, None]
+ (kx.reshape(1, 1, 9, 1, 1) - 1) * dilation[1]
+ delta[:, :, :, 1]
)
grid = torch.stack(
(((xcoord + 0.5) * 2 / width - 1), ((y + 0.5) * 2 / height - 1)), dim=-1
)
grid = grid.permute(0, 1, 3, 4, 2, 5).reshape(batch * groups, height, width * 9, 2)
sampled = F.grid_sample(
x.reshape(batch * groups, channels // groups, height, width),
grid.to(dtype),
mode="bilinear",
padding_mode="zeros",
align_corners=False,
)
sampled = sampled.reshape(batch, groups, channels // groups, height, width, 9)
sampled = sampled.permute(0, 1, 2, 5, 3, 4)
if mask is not None:
sampled = sampled * mask.reshape(batch, groups, 1, 9, height, width).to(dtype)
columns = sampled.reshape(batch, channels * 9, height, width)
# The column tensor is already grouped channel-major, so the 1x1 convolution
# runs with groups=1 over the flattened (channel, kernel-position) axis.
return F.conv2d(
columns,
weight.reshape(weight.shape[0], weight.shape[1] * 9, 1, 1),
bias,
)
def box_iou(boxes, box):
"""IoU of every row in ``boxes`` [N,4] against one ``box`` [4] (xyxy)."""
lower = torch.maximum(boxes[:, 0:2], box[0:2])
upper = torch.minimum(boxes[:, 2:4], box[2:4])
wh = (upper - lower).clamp(min=0)
inter = wh[:, 0] * wh[:, 1]
area_boxes = (boxes[:, 2:4] - boxes[:, 0:2]).clamp(min=0).prod(dim=1)
area_box = (box[2:4] - box[0:2]).clamp(min=0).prod()
union = area_boxes + area_box - inter
return torch.where(union > 0, inter / union, torch.zeros_like(inter))
@torch.no_grad()
def nms(boxes, scores, iou_threshold):
"""Greedy NMS matching ``torchvision.ops.nms`` ordering.
torchvision sorts by descending score and keeps the first box when IoU is
strictly greater than the threshold; equal scores keep their original order.
"""
if boxes.numel() == 0:
return torch.empty((0,), dtype=torch.long, device=boxes.device)
order = torch.argsort(scores, descending=True, stable=True)
keep = []
while order.numel():
index = order[0]
keep.append(index)
if order.numel() == 1:
break
iou = box_iou(boxes[order[1:]], boxes[index])
order = order[1:][iou <= iou_threshold]
return torch.stack(keep)
def _same_padding(kernel_size, stride=1):
return ((stride - 1) + kernel_size - 1) // 2
class ConvBNAct(nn.Module):
"""Conv + BatchNorm + optional ReLU, the HGNetv2 basic block."""
def __init__(self, in_chs, out_chs, kernel_size, stride=1, groups=1, use_act=True):
super().__init__()
self.conv = nn.Conv2d(
in_chs, out_chs, kernel_size, stride=stride,
padding=_same_padding(kernel_size, stride), groups=groups, bias=False,
)
self.bn = nn.BatchNorm2d(out_chs)
self.act = nn.ReLU() if use_act else nn.Identity()
def forward(self, x):
return self.act(self.bn(self.conv(x)))
class LightConvBNAct(nn.Module):
"""1x1 pointwise conv followed by a depthwise conv."""
def __init__(self, in_chs, out_chs, kernel_size):
super().__init__()
self.conv1 = ConvBNAct(in_chs, out_chs, kernel_size=1, use_act=False)
self.conv2 = ConvBNAct(out_chs, out_chs, kernel_size=kernel_size, groups=out_chs)
def forward(self, x):
return self.conv2(self.conv1(x))
class Stem(nn.Module):
"""PP-HGNetv2 stem: two stride-2 stages reaching stride 4."""
def __init__(self, in_chs=3, mid_chs=32, out_chs=48):
super().__init__()
self.stem1 = ConvBNAct(in_chs, mid_chs, kernel_size=3, stride=2)
self.stem2a = ConvBNAct(mid_chs, mid_chs // 2, kernel_size=2, stride=1)
self.stem2b = ConvBNAct(mid_chs // 2, mid_chs, kernel_size=2, stride=1)
self.stem3 = ConvBNAct(mid_chs * 2, mid_chs, kernel_size=3, stride=2)
self.stem4 = ConvBNAct(mid_chs, out_chs, kernel_size=1, stride=1)
self.pool = nn.MaxPool2d(kernel_size=2, stride=1, ceil_mode=True)
def forward(self, x):
x = self.stem1(x)
x = F.pad(x, (0, 1, 0, 1))
# Two separate pads: the pooled branch sees the first one only, the
# convolution branch both. Collapsing them changes the spatial size.
branch = F.pad(self.stem2a(x), (0, 1, 0, 1))
branch = self.stem2b(branch)
pooled = self.pool(x)
return self.stem4(self.stem3(torch.cat([pooled, branch], dim=1)))
class DenseBlock(nn.Module):
"""Every layer's output is concatenated, then squeezed by two 1x1 convs."""
def __init__(self, in_chs, mid_chs, out_chs, layer_num, kernel_size=3,
residual=False, light_block=False):
super().__init__()
self.residual = residual
self.layers = nn.ModuleList(
(LightConvBNAct if light_block else ConvBNAct)(
in_chs if index == 0 else mid_chs, mid_chs, kernel_size=kernel_size
)
for index in range(layer_num)
)
total_chs = in_chs + layer_num * mid_chs
self.aggregation = nn.Sequential(
ConvBNAct(total_chs, out_chs // 2, kernel_size=1),
ConvBNAct(out_chs // 2, out_chs, kernel_size=1),
)
def forward(self, x):
identity = x
collected = [x]
for layer in self.layers:
x = layer(x)
collected.append(x)
x = self.aggregation(torch.cat(collected, dim=1))
return x + identity if self.residual else x
class DenseStage(nn.Module):
def __init__(self, in_chs, mid_chs, out_chs, block_num, layer_num,
downsample=True, stride=2, light_block=False, kernel_size=3):
super().__init__()
self.downsample = (
ConvBNAct(in_chs, in_chs, kernel_size=3, stride=stride,
groups=in_chs, use_act=False)
if downsample else nn.Identity()
)
self.blocks = nn.Sequential(*[
DenseBlock(
in_chs if index == 0 else out_chs,
mid_chs,
out_chs,
layer_num,
residual=index != 0,
kernel_size=kernel_size,
light_block=light_block,
)
for index in range(block_num)
])
def forward(self, x):
return self.blocks(self.downsample(x))
HGNETV2_B4_STAGES = (
(48, 48, 128, 1, False, False, 3, 6),
(128, 96, 512, 1, True, False, 3, 6),
(512, 192, 1024, 3, True, True, 5, 6),
)
class HGNetv2B4Features(nn.Module):
"""HGNetv2-b4 truncated after stage 2: stride 16, 1024 channels.
Stages are plain attributes named ``stages_0`` ... because that is what the
checkpoint calls them (timm's ``flatten_sequential`` renames ``stages.0``).
"""
out_channels = 1024
def __init__(self):
super().__init__()
self.stem = Stem(in_chs=3, mid_chs=32, out_chs=48)
for index, stage in enumerate(HGNETV2_B4_STAGES):
setattr(
self,
f"stages_{index}",
DenseStage(
in_chs=stage[0],
mid_chs=stage[1],
out_chs=stage[2],
block_num=stage[3],
layer_num=stage[7],
downsample=stage[4],
light_block=stage[5],
kernel_size=stage[6],
),
)
self.stage_names = tuple(f"stages_{index}" for index in range(len(HGNETV2_B4_STAGES)))
class DeformableAdapter(nn.Module):
"""Identity-initialised bottleneck with one DCNv2 at its centre."""
def __init__(self, channels, bottleneck_channels):
super().__init__()
self.reduce = nn.Conv2d(channels, bottleneck_channels, 1, bias=False)
self.bn1 = nn.BatchNorm2d(bottleneck_channels)
self.dcn = DCNv2(bottleneck_channels, deform_groups=2)
self.bn2 = nn.BatchNorm2d(bottleneck_channels)
self.expand = nn.Conv2d(bottleneck_channels, channels, 1, bias=False)
self.bn3 = nn.BatchNorm2d(channels)
self.relu = nn.ReLU(inplace=True)
def forward(self, x):
residual = self.relu(self.bn1(self.reduce(x)))
residual = self.relu(self.bn2(self.dcn(residual)))
return x + self.bn3(self.expand(residual))
class HGNetv2B4Backbone(nn.Module):
"""HGNetv2-b4 plus the two deformable adapters."""
def __init__(self, deformable=True):
super().__init__()
self.deformable = deformable
self.features = HGNetv2B4Features()
self.out_channels = HGNetv2B4Features.out_channels
self.dcn_adapters = nn.ModuleDict()
if deformable:
self.dcn_adapters["stages_1"] = DeformableAdapter(512, 128)
self.dcn_adapters["stages_2"] = DeformableAdapter(1024, 256)
def forward(self, images):
if not self.deformable:
return self.features(images)
x = self.features.stem(images)
for name in self.features.stage_names:
x = getattr(self.features, name)(x)
if name in self.dcn_adapters:
x = self.dcn_adapters[name](x)
return x
class ConvAct(nn.Module):
def __init__(self, in_channels, out_channels, kernel_size=3, activation=True):
super().__init__()
self.conv = nn.Conv2d(in_channels, out_channels, kernel_size,
padding=kernel_size // 2)
self.activate = nn.ReLU(inplace=True) if activation else nn.Identity()
def forward(self, x):
return self.activate(self.conv(x))
class NeckProjection(nn.Module):
"""Lateral conv then output conv (the only active FPN legs in the checkpoint)."""
def __init__(self, in_channels, out_channels=256):
super().__init__()
self.lateral_convs = nn.ModuleList([ConvAct(in_channels, out_channels, 1, False)])
self.fpn_convs = nn.ModuleList([ConvAct(out_channels, out_channels, 3, False)])
def forward(self, x):
return self.fpn_convs[0](self.lateral_convs[0](x))
class PolygonHead(nn.Module):
"""Dense/sparse class logits plus 14 low-rank polygon coefficients."""
def __init__(self, in_channels=256, num_coefficients=14, upsample=4):
super().__init__()
self.upsample = upsample
self.cls_convs = nn.Sequential(ConvAct(in_channels, 32), ConvAct(32, 32))
self.out_conv_cls_dense = nn.Conv2d(32, 1, 3, padding=1)
self.out_conv_reg_dense = nn.Conv2d(32, num_coefficients, 3, padding=1)
self.out_conv_cls_sparse = nn.Conv2d(32, 1, 3, padding=1)
self.out_conv_reg_sparse = nn.Conv2d(32, num_coefficients, 3, padding=1)
def forward(self, feature):
feature = F.interpolate(feature, scale_factor=self.upsample,
mode="bilinear", align_corners=False)
feature = self.cls_convs(feature)
# ``out_conv_reg_dense`` is kept for checkpoint compatibility but never
# read: decoding uses only the sparse branch. At 1600x1600 input this is
# a 400x400x14 convolution that was pure waste.
return {
"dense_logits": self.out_conv_cls_dense(feature),
"sparse_logits": self.out_conv_cls_sparse(feature),
"sparse_coefficients": self.out_conv_reg_sparse(feature),
}
class DetectorNet(nn.Module):
"""Backbone, projection and head, in the order the frozen detector runs them."""
def __init__(self, num_coefficients=14, deformable=True, upsample=4):
super().__init__()
if upsample not in (1, 2, 4):
raise ValueError("upsample must be 1, 2 or 4")
self.output_stride = 16 // upsample
self.backbone = HGNetv2B4Backbone(deformable=deformable)
self.neck = NeckProjection(self.backbone.out_channels)
self.bbox_head = PolygonHead(num_coefficients=num_coefficients, upsample=upsample)
def forward(self, images):
if images.ndim != 4 or any(size % 32 for size in images.shape[-2:]):
raise ValueError("Expected BCHW images padded to multiples of 32")
return self.bbox_head(self.neck(self.backbone(images)))
def polygon_points(coefficients, basis, stride):
"""``[B,K,H,W]`` coefficients -> ``[B,H*W,P,2]`` pixel coords.
"""
batch, _, height, width = coefficients.shape
basis = torch.as_tensor(basis, dtype=torch.float32, device=coefficients.device)
points = coefficients.permute(0, 2, 3, 1).reshape(batch, height * width, -1) @ basis
yy, xx = torch.meshgrid(
torch.arange(height, device=coefficients.device, dtype=torch.float32),
torch.arange(width, device=coefficients.device, dtype=torch.float32),
indexing="ij",
)
locations = torch.stack((xx.reshape(-1), yy.reshape(-1)), dim=1)
locations = locations.repeat(1, basis.shape[1] // 2)
return (points + locations) * stride
@torch.no_grad()
def _drop_degenerate(polygons, content_shape, config):
"""Mask out boxes too small to hold readable text.
The optional area cap is disabled by default -- see the comment below for
why "covers half the image" is not a reliable signal.
"""
flat = polygons.reshape(len(polygons), -1, 2)
wh = (flat.amax(dim=1) - flat.amin(dim=1)).clamp(min=0)
min_side = float(config.get("min_box_side", 4.0))
keep = (wh[:, 0] >= min_side) & (wh[:, 1] >= min_side)
# Area cap is OFF by default: a single-line crop legitimately fills most of
# its own image, so "covers half the page" is not evidence of a bad box.
# The few page-sized regions it would catch are not worth deleting whole
# lines from cropped inputs.
fraction = float(config.get("max_box_area_fraction", 1.0))
if content_shape is not None and fraction < 1.0:
height, width = content_shape
keep = keep & ((wh[:, 0] * wh[:, 1]) <= fraction * float(height * width))
return keep
def _suppress_contained(polygons, scores, contain=0.95, ratio=0.35):
"""Drop boxes nested inside a larger box.
Measured on 3544 pages: 54% of images contain nested boxes and 40% of them
repeat text already returned by the enclosing line. Containment (not IoU)
is the right test -- a small box inside a large one has a low IoU, so an
IoU filter never fires.
The ratio guard keeps legitimate nesting: a heading inside a paragraph block
is usually more than a third of its area, so it survives.
"""
n = len(polygons)
if n < 2:
return polygons, scores
flat = polygons.reshape(n, -1, 2)
lo, hi = flat.amin(dim=1), flat.amax(dim=1)
areas = (hi - lo).clamp(min=0).prod(dim=1)
order = torch.argsort(areas, descending=True)
lo, hi, areas = lo[order], hi[order], areas[order]
alive = torch.ones(n, dtype=torch.bool, device=areas.device)
for pos in range(n - 1):
if not bool(alive[pos]):
continue
candidate = alive[pos + 1:]
if not bool(candidate.any()):
continue
c_lo, c_hi, c_area = lo[pos + 1:], hi[pos + 1:], areas[pos + 1:]
ix0 = torch.maximum(lo[pos][0], c_lo[:, 0])
iy0 = torch.maximum(lo[pos][1], c_lo[:, 1])
ix1 = torch.minimum(hi[pos][0], c_hi[:, 0])
iy1 = torch.minimum(hi[pos][1], c_hi[:, 1])
inter = (ix1 - ix0).clamp(min=0) * (iy1 - iy0).clamp(min=0)
nested = (inter / c_area.clamp(min=1e-6) >= contain) & (
c_area / areas[pos].clamp(min=1e-6) <= ratio
)
alive[pos + 1:] = candidate & ~nested
keep = torch.zeros(n, dtype=torch.bool, device=areas.device)
keep[order] = alive
return polygons[keep], scores[keep]
def decode_detection(predictions, basis, config, content_shape=None, scale=None):
"""Scores plus polygons to accepted boxes, mirroring the frozen decoder.
``config`` needs ``stride``, ``score_threshold``, ``nms_threshold`` and
``max_candidates``. ``content_shape`` (h, w pre-padding) masks padding
cells; ``scale`` divides polygons back to original image coordinates.
"""
stride = config["stride"]
points = polygon_points(
predictions["sparse_coefficients"].float(), basis, stride
)
confidence = (
predictions["dense_logits"].sigmoid()
* predictions["sparse_logits"].sigmoid()
)[:, 0]
if content_shape is not None:
height, width = content_shape
confidence = confidence.clone()
confidence[:, (height + stride - 1) // stride :] = 0
confidence[:, :, (width + stride - 1) // stride :] = 0
results = []
for polygons, scores in zip(points, confidence.flatten(1)):
selected = (scores > config["score_threshold"]).nonzero(as_tuple=True)[0]
if selected.numel() > config["max_candidates"]:
selected = selected[
scores[selected].topk(config["max_candidates"]).indices
]
polygons, scores = polygons[selected], scores[selected]
if polygons.numel():
flat = polygons.reshape(len(polygons), -1, 2)
boxes = torch.cat((flat.amin(dim=1), flat.amax(dim=1)), dim=1)
keep = nms(boxes, scores, config["nms_threshold"])
polygons, scores = polygons[keep], scores[keep]
# Added post-processing, all geometry-only so it runs before recognition.
keep = _drop_degenerate(polygons, content_shape, config)
polygons, scores = polygons[keep], scores[keep]
if polygons.numel() and config.get("deduplicate_nested", False):
polygons, scores = _suppress_contained(
polygons, scores,
float(config.get("contain_threshold", 0.95)),
float(config.get("contain_area_ratio", 0.35)),
)
if content_shape is not None and polygons.numel():
# Clamp to the content area, not the padded canvas: a polygon
# that runs into the padding would otherwise be cropped with
# replicated border pixels.
content_h, content_w = content_shape
view = polygons.reshape(len(polygons), -1, 2)
view[..., 0].clamp_(0, content_w - 1)
view[..., 1].clamp_(0, content_h - 1)
if scale is not None:
polygons = polygons / torch.as_tensor(
scale, dtype=polygons.dtype, device=polygons.device
).repeat(polygons.shape[-1] // 2)
results.append(
{"polygons": polygons.cpu().numpy(), "scores": scores.cpu().numpy()}
)
return results
MEAN = np.array([123.675, 116.28, 103.53], dtype=np.float32)
STD = np.array([58.395, 57.12, 57.375], dtype=np.float32)
def detection_tensor(rgb, size=1600):
"""Letterbox one RGB uint8 image to ``size`` and normalise.
Returns ``(tensor [1,3,size,size] float32, scale [2], content (h, w))``.
"""
height, width = rgb.shape[:2]
scale = size / max(height, width)
new_w = max(1, round(width * scale))
new_h = max(1, round(height * scale))
resized = np.asarray(Image.fromarray(rgb).resize((new_w, new_h), Image.BILINEAR))
canvas = np.zeros((size, size, 3), np.float32)
canvas[:new_h, :new_w] = (resized.astype(np.float32) - MEAN) / STD
tensor = np.ascontiguousarray(canvas.transpose(2, 0, 1)[None])
return tensor, np.array([new_w / width, new_h / height], np.float32), (new_h, new_w)
def _require_cv2():
"""Import OpenCV on demand.
``transformers`` statically scans a ``trust_remote_code`` module for imports
and refuses to load it when any of them is missing, so a top-level
``import cv2`` would make the detector unloadable on a machine that only
wants ``detect()``. The import sits inside a ``try`` block because the
scanner does not descend into those.
"""
try:
import cv2
except ImportError as exc: # pragma: no cover - depends on the environment
raise ImportError(
"OpenCV is required for polygon rectification (det.ocr / the crop "
"step). Install it with `pip install opencv-python-headless` -- the "
"transformers hint `pip install cv2` refers to a package that does "
"not exist. Detection alone (det.detect) works without it."
) from exc
return cv2
def polygon_to_quad(polygon, image_shape):
"""Return TL/TR/BR/BL float32 corners compatible with OpenOCR's cropper.
Uses OpenOCR's minAreaRect corner convention (sort by x, then split into
left/right pairs), without the int16 coordinate limit.
"""
points = np.asarray(polygon, dtype=np.float32).reshape(-1, 2).copy()
height, width = image_shape[:2]
if len(points) < 3 or not np.isfinite(points).all() or min(height, width) < 2:
return None
cv2 = _require_cv2()
points[:, 0] = np.clip(points[:, 0], 0, width - 1)
points[:, 1] = np.clip(points[:, 1], 0, height - 1)
rect = cv2.minAreaRect(points)
if min(rect[1]) < 2:
return None
corners = cv2.boxPoints(rect)
corners = corners[np.argsort(corners[:, 0], kind="stable")]
left = corners[:2][np.argsort(corners[:2, 1], kind="stable")]
right = corners[2:][np.argsort(corners[2:, 1], kind="stable")]
quad = np.asarray([left[0], right[0], right[1], left[1]], dtype=np.float32)
quad[:, 0] = np.clip(quad[:, 0], 0, width - 1)
quad[:, 1] = np.clip(quad[:, 1], 0, height - 1)
crop_width = max(
np.linalg.norm(quad[0] - quad[1]), np.linalg.norm(quad[2] - quad[3])
)
crop_height = max(
np.linalg.norm(quad[0] - quad[3]), np.linalg.norm(quad[1] - quad[2])
)
if min(crop_width, crop_height) < 2 or abs(cv2.contourArea(quad)) < 1:
return None
return quad
def get_rotate_crop_image(img, points):
"""Four-point perspective crop, identical to the frozen OpenOCR cropper.
Tall crops (height / width >= 1.5) are rotated 90 degrees so vertical text
is fed horizontally; dropping that step costs accuracy on vertical samples.
"""
points = np.asarray(points, dtype=np.float32).reshape(4, 2)
cv2 = _require_cv2()
img_crop_width = int(
max(np.linalg.norm(points[0] - points[1]), np.linalg.norm(points[2] - points[3]))
)
img_crop_height = int(
max(np.linalg.norm(points[0] - points[3]), np.linalg.norm(points[1] - points[2]))
)
pts_std = np.float32(
[
[0, 0],
[img_crop_width, 0],
[img_crop_width, img_crop_height],
[0, img_crop_height],
]
)
matrix = cv2.getPerspectiveTransform(points, pts_std)
cropped = cv2.warpPerspective(
img,
matrix,
(img_crop_width, img_crop_height),
borderMode=cv2.BORDER_REPLICATE,
flags=cv2.INTER_CUBIC,
)
height, width = cropped.shape[0:2]
if width and height * 1.0 / width >= 1.5:
cropped = np.rot90(cropped)
return cropped
class OCRPipeline:
"""Detector plus one recogniser, in the frozen pipeline's order.
Detection runs per image, polygons become upright quads, crops are sorted
and bucketed by recognition shape, and regions come back in reading order
(top-to-bottom, then left-to-right of each quad's top-left corner).
"""
def __init__(self, det_model, rec_model, batch_size=32, drop_score=0.0):
self.det_model = det_model
self.rec_model = rec_model
self.batch_size = batch_size
self.drop_score = drop_score
def __call__(self, image, color="rgb"):
"""``image`` is a path, bytes, PIL image or HxWx3 array.
Returns ``{"text": [str], "regions": [{text, score, quad, ...}]}``.
"""
return self.run(self._read(image, color))
def run(self, rgb):
regions = self.detect(rgb)
if not regions:
return {"text": [], "regions": []}
# ``recognition_batches`` expects BGR crops and converts them to RGB
# itself, so the crop stays in BGR rather than being flipped back here.
bgr = np.ascontiguousarray(rgb[:, :, ::-1])
crops = [get_rotate_crop_image(bgr, region["quad"]) for region in regions]
recognised = self.recognize(crops)
keep = []
for region, (text, score) in zip(regions, recognised):
if score is not None and not np.isnan(score) and score < self.drop_score:
continue
region = dict(region)
region["text"] = text
region["rec_score"] = None if score is None else float(score)
keep.append(region)
return {"text": [region["text"] for region in keep], "regions": keep}
def detect(self, rgb):
raw = self.det_model.detect([rgb], batch_size=1)[0]
regions = []
for polygon, score in zip(raw["polygons"], raw["scores"]):
quad = polygon_to_quad(polygon, rgb.shape)
if quad is None:
continue
regions.append(
{"polygon": np.asarray(polygon), "quad": quad, "det_score": float(score)}
)
return sorted(regions, key=lambda row: (row["quad"][0, 1], row["quad"][0, 0]))
def recognize(self, crops):
return self.rec_model.recognize(crops, batch_size=self.batch_size)
@staticmethod
def _read(image, color):
from PIL import Image, ImageOps
def _to_rgb(im):
"""Composite alpha over white, apply EXIF orientation, return RGB."""
im = ImageOps.exif_transpose(im)
if im.mode in ("RGBA", "LA") or "A" in im.mode:
bg = Image.new("RGBA", im.size, (255, 255, 255, 255))
im = Image.alpha_composite(bg, im.convert("RGBA")).convert("RGB")
elif im.mode == "P":
im = im.convert("RGBA")
bg = Image.new("RGBA", im.size, (255, 255, 255, 255))
im = Image.alpha_composite(bg, im).convert("RGB")
else:
im = im.convert("RGB")
return im
if isinstance(image, np.ndarray):
if image.ndim == 2:
image = np.stack([image] * 3, axis=-1)
if color == "bgr":
return np.ascontiguousarray(image[:, :, ::-1])
return np.ascontiguousarray(image)
if isinstance(image, (bytes, bytearray)):
import io
with Image.open(io.BytesIO(image)) as im:
return np.asarray(_to_rgb(im))
if isinstance(image, (str, bytes)) or hasattr(image, "__fspath__"):
with Image.open(image) as im:
return np.asarray(_to_rgb(im))
return np.asarray(_to_rgb(image))
class OpenOCRv2DetConfig(PretrainedConfig):
"""Configuration for the OpenOCRv2 detector (``OpenOCRv2_Det``).
The defaults are the frozen post-processing constants from
``models/manifest.json`` (``models.det``); overriding them changes which
boxes survive, not just the scores.
"""
model_type = "openocrv2_det"
def __init__(
self,
image_size=1600,
stride=4,
score_threshold=0.2,
nms_threshold=0.5,
max_candidates=2000,
num_coefficients=14,
upsample=4,
deformable=True,
architecture="hgnetv2_b4",
**kwargs,
):
super().__init__(**kwargs)
self.image_size = image_size
self.stride = stride
self.score_threshold = score_threshold
self.nms_threshold = nms_threshold
self.max_candidates = max_candidates
self.num_coefficients = num_coefficients
self.upsample = upsample
self.deformable = deformable
self.architecture = architecture
@property
def decode_config(self):
return {
"stride": self.stride,
"score_threshold": self.score_threshold,
"nms_threshold": self.nms_threshold,
"max_candidates": self.max_candidates,
# Nested-box suppression is OFF by default. It removes text that a
# downstream metric may expect, and ANLS never rewards removing it
# (measured: -0.43 points on a 54-image ablation). Turn it on for
# pseudo-labelling, where the goal is clean boxes for annotators.
"deduplicate_nested": getattr(self, "deduplicate_nested", False),
"contain_threshold": getattr(self, "contain_threshold", 0.95),
"contain_area_ratio": getattr(self, "contain_area_ratio", 0.35),
"min_box_side": getattr(self, "min_box_side", 4.0),
}
@dataclass
class OpenOCRv2DetOutput(ModelOutput):
"""Raw detection head outputs plus absolute pixel polygons."""
dense_logits: torch.FloatTensor | None = None
sparse_logits: torch.FloatTensor | None = None
sparse_coefficients: torch.FloatTensor | None = None
class OpenOCRv2DetModel(PreTrainedModel):
"""OpenOCRv2 detector: dense/sparse logits plus 14 polygon coefficients."""
config_class = OpenOCRv2DetConfig
main_input_name = "pixel_values"
def __init__(self, config):
super().__init__(config)
self.model = DetectorNet(
num_coefficients=config.num_coefficients,
deformable=config.deformable,
upsample=config.upsample,
)
# Polygon basis (14 x 28). A buffer, so it travels with the checkpoint.
self.register_buffer(
"basis", torch.zeros(config.num_coefficients, config.num_coefficients * 2)
)
self.post_init()
def forward(self, pixel_values, **kwargs):
predictions = self.model(pixel_values)
return OpenOCRv2DetOutput(
dense_logits=predictions["dense_logits"],
sparse_logits=predictions["sparse_logits"],
sparse_coefficients=predictions["sparse_coefficients"],
)
def detect(self, images, batch_size=1):
"""Run detection on a list of RGB uint8 arrays.
Returns one dict per image with ``polygons``, ``quads``, ``scores`` and the
polygons expressed in original-image pixels.
"""
self.eval()
results = []
device = next(self.parameters()).device
for start in range(0, len(images), batch_size):
chunk = images[start : start + batch_size]
tensors, scales, contents = [], [], []
for rgb in chunk:
tensor, scale, content = detection_tensor(
rgb, self.config.image_size
)
tensors.append(tensor[0])
scales.append(scale)
contents.append(content)
pixel_values = torch.from_numpy(np.stack(tensors)).to(device)
with torch.no_grad():
outputs = self(pixel_values=pixel_values)
for index, (scale, content) in enumerate(zip(scales, contents)):
single = {
"dense_logits": outputs.dense_logits[index : index + 1],
"sparse_logits": outputs.sparse_logits[index : index + 1],
"sparse_coefficients": outputs.sparse_coefficients[index : index + 1],
}
decoded = decode_detection(
single,
self.basis,
self.config.decode_config,
content_shape=content,
scale=scale,
)[0]
results.append(decoded)
return results
def ocr(self, image, rec_model, batch_size=32, drop_score=0.0):
"""End-to-end OCR on one page; ``rec_model`` is one of the recognizers.
Detect, rectify every polygon into an upright crop, recognise, and return
the regions in reading order. This is the same sequence as
``OCRPipeline`` -- it just saves the caller from tying the two halves
together (and from re-implementing the cropper, which is where accuracy
is most easily lost).
``image`` is a path, bytes, PIL image or HxWx3 uint8 array.
Returns ``{"text": [str], "regions": [{text, rec_score, det_score, quad}]}``.
"""
return OCRPipeline(
self, rec_model, batch_size=batch_size, drop_score=drop_score
)(image)