import numpy as np import torch from torch import nn from torch.nn import functional as F from transformers import PretrainedConfig from transformers import PreTrainedModel from transformers.modeling_outputs import ModelOutput from PIL import Image from dataclasses import dataclass class DCNv2(nn.Module): """Modulated deformable 3x3 convolution with learned offsets and masks. Parameter names match the frozen checkpoint (``weight``, ``conv_offset.*``). ``forward`` dispatches to the portable path so this class is importable without torchvision. """ def __init__(self, channels, stride=1, deform_groups=2): super().__init__() self.in_channels = channels self.out_channels = channels self.stride = (stride, stride) self.padding = (1, 1) self.dilation = (1, 1) self.deform_groups = deform_groups # Registered the same way nn.Conv2d would, so key names line up. self.weight = nn.Parameter(torch.empty(channels, channels, 3, 3)) self.bias = None self.conv_offset = nn.Conv2d( channels, deform_groups * 27, 3, stride=stride, padding=1 ) def forward(self, x): offset_y, offset_x, mask = self.conv_offset(x).chunk(3, dim=1) return deform_conv2d_portable( x, torch.cat((offset_y, offset_x), dim=1), self.weight, self.bias, self.stride, self.padding, self.dilation, mask=mask.sigmoid(), groups=self.deform_groups, ) def deform_conv2d_portable( x, offset, weight, bias=None, stride=(1, 1), padding=(1, 1), dilation=(1, 1), mask=None, groups=1, ): """Modulated deformable convolution via ``grid_sample``. ``offset`` has ``2 * groups * 9`` channels ordered as ``[group][ky][kx][dy, dx]`` — the layout ``torchvision`` expects and the one the trained ``conv_offset`` produces. """ if isinstance(stride, int): stride = (stride, stride) if isinstance(padding, int): padding = (padding, padding) if isinstance(dilation, int): dilation = (dilation, dilation) if (stride, padding, dilation) != ((1, 1), (1, 1), (1, 1)): raise ValueError( "Portable DCN covers the frozen stride-1, pad-1, dilation-1 adapters only" ) batch, channels, height, width = x.shape if offset.shape[1] != groups * 18: raise ValueError( f"offset has {offset.shape[1]} channels; expected {groups * 18} " f"for {groups} deform groups" ) dtype = x.dtype device = x.device delta = offset.reshape(batch, groups, 9, 2, height, width) yy, xx = torch.meshgrid( torch.arange(height, device=device, dtype=torch.float32), torch.arange(width, device=device, dtype=torch.float32), indexing="ij", ) ky, kx = torch.meshgrid( torch.arange(3, device=device, dtype=torch.float32), torch.arange(3, device=device, dtype=torch.float32), indexing="ij", ) # Coordinates are expressed in the *unpadded* input frame; padding is handled # by ``padding_mode="zeros"``. Adding a padding offset here would shift every # sample by ``padding`` pixels (verified against torchvision: it does not). y = yy[None, None, None] + (ky.reshape(1, 1, 9, 1, 1) - 1) * dilation[0] + delta[:, :, :, 0] xcoord = ( xx[None, None, None] + (kx.reshape(1, 1, 9, 1, 1) - 1) * dilation[1] + delta[:, :, :, 1] ) grid = torch.stack( (((xcoord + 0.5) * 2 / width - 1), ((y + 0.5) * 2 / height - 1)), dim=-1 ) grid = grid.permute(0, 1, 3, 4, 2, 5).reshape(batch * groups, height, width * 9, 2) sampled = F.grid_sample( x.reshape(batch * groups, channels // groups, height, width), grid.to(dtype), mode="bilinear", padding_mode="zeros", align_corners=False, ) sampled = sampled.reshape(batch, groups, channels // groups, height, width, 9) sampled = sampled.permute(0, 1, 2, 5, 3, 4) if mask is not None: sampled = sampled * mask.reshape(batch, groups, 1, 9, height, width).to(dtype) columns = sampled.reshape(batch, channels * 9, height, width) # The column tensor is already grouped channel-major, so the 1x1 convolution # runs with groups=1 over the flattened (channel, kernel-position) axis. return F.conv2d( columns, weight.reshape(weight.shape[0], weight.shape[1] * 9, 1, 1), bias, ) def box_iou(boxes, box): """IoU of every row in ``boxes`` [N,4] against one ``box`` [4] (xyxy).""" lower = torch.maximum(boxes[:, 0:2], box[0:2]) upper = torch.minimum(boxes[:, 2:4], box[2:4]) wh = (upper - lower).clamp(min=0) inter = wh[:, 0] * wh[:, 1] area_boxes = (boxes[:, 2:4] - boxes[:, 0:2]).clamp(min=0).prod(dim=1) area_box = (box[2:4] - box[0:2]).clamp(min=0).prod() union = area_boxes + area_box - inter return torch.where(union > 0, inter / union, torch.zeros_like(inter)) @torch.no_grad() def nms(boxes, scores, iou_threshold): """Greedy NMS matching ``torchvision.ops.nms`` ordering. torchvision sorts by descending score and keeps the first box when IoU is strictly greater than the threshold; equal scores keep their original order. """ if boxes.numel() == 0: return torch.empty((0,), dtype=torch.long, device=boxes.device) order = torch.argsort(scores, descending=True, stable=True) keep = [] while order.numel(): index = order[0] keep.append(index) if order.numel() == 1: break iou = box_iou(boxes[order[1:]], boxes[index]) order = order[1:][iou <= iou_threshold] return torch.stack(keep) def _same_padding(kernel_size, stride=1): return ((stride - 1) + kernel_size - 1) // 2 class ConvBNAct(nn.Module): """Conv + BatchNorm + optional ReLU, the HGNetv2 basic block.""" def __init__(self, in_chs, out_chs, kernel_size, stride=1, groups=1, use_act=True): super().__init__() self.conv = nn.Conv2d( in_chs, out_chs, kernel_size, stride=stride, padding=_same_padding(kernel_size, stride), groups=groups, bias=False, ) self.bn = nn.BatchNorm2d(out_chs) self.act = nn.ReLU() if use_act else nn.Identity() def forward(self, x): return self.act(self.bn(self.conv(x))) class LightConvBNAct(nn.Module): """1x1 pointwise conv followed by a depthwise conv.""" def __init__(self, in_chs, out_chs, kernel_size): super().__init__() self.conv1 = ConvBNAct(in_chs, out_chs, kernel_size=1, use_act=False) self.conv2 = ConvBNAct(out_chs, out_chs, kernel_size=kernel_size, groups=out_chs) def forward(self, x): return self.conv2(self.conv1(x)) class Stem(nn.Module): """PP-HGNetv2 stem: two stride-2 stages reaching stride 4.""" def __init__(self, in_chs=3, mid_chs=32, out_chs=48): super().__init__() self.stem1 = ConvBNAct(in_chs, mid_chs, kernel_size=3, stride=2) self.stem2a = ConvBNAct(mid_chs, mid_chs // 2, kernel_size=2, stride=1) self.stem2b = ConvBNAct(mid_chs // 2, mid_chs, kernel_size=2, stride=1) self.stem3 = ConvBNAct(mid_chs * 2, mid_chs, kernel_size=3, stride=2) self.stem4 = ConvBNAct(mid_chs, out_chs, kernel_size=1, stride=1) self.pool = nn.MaxPool2d(kernel_size=2, stride=1, ceil_mode=True) def forward(self, x): x = self.stem1(x) x = F.pad(x, (0, 1, 0, 1)) # Two separate pads: the pooled branch sees the first one only, the # convolution branch both. Collapsing them changes the spatial size. branch = F.pad(self.stem2a(x), (0, 1, 0, 1)) branch = self.stem2b(branch) pooled = self.pool(x) return self.stem4(self.stem3(torch.cat([pooled, branch], dim=1))) class DenseBlock(nn.Module): """Every layer's output is concatenated, then squeezed by two 1x1 convs.""" def __init__(self, in_chs, mid_chs, out_chs, layer_num, kernel_size=3, residual=False, light_block=False): super().__init__() self.residual = residual self.layers = nn.ModuleList( (LightConvBNAct if light_block else ConvBNAct)( in_chs if index == 0 else mid_chs, mid_chs, kernel_size=kernel_size ) for index in range(layer_num) ) total_chs = in_chs + layer_num * mid_chs self.aggregation = nn.Sequential( ConvBNAct(total_chs, out_chs // 2, kernel_size=1), ConvBNAct(out_chs // 2, out_chs, kernel_size=1), ) def forward(self, x): identity = x collected = [x] for layer in self.layers: x = layer(x) collected.append(x) x = self.aggregation(torch.cat(collected, dim=1)) return x + identity if self.residual else x class DenseStage(nn.Module): def __init__(self, in_chs, mid_chs, out_chs, block_num, layer_num, downsample=True, stride=2, light_block=False, kernel_size=3): super().__init__() self.downsample = ( ConvBNAct(in_chs, in_chs, kernel_size=3, stride=stride, groups=in_chs, use_act=False) if downsample else nn.Identity() ) self.blocks = nn.Sequential(*[ DenseBlock( in_chs if index == 0 else out_chs, mid_chs, out_chs, layer_num, residual=index != 0, kernel_size=kernel_size, light_block=light_block, ) for index in range(block_num) ]) def forward(self, x): return self.blocks(self.downsample(x)) HGNETV2_B4_STAGES = ( (48, 48, 128, 1, False, False, 3, 6), (128, 96, 512, 1, True, False, 3, 6), (512, 192, 1024, 3, True, True, 5, 6), ) class HGNetv2B4Features(nn.Module): """HGNetv2-b4 truncated after stage 2: stride 16, 1024 channels. Stages are plain attributes named ``stages_0`` ... because that is what the checkpoint calls them (timm's ``flatten_sequential`` renames ``stages.0``). """ out_channels = 1024 def __init__(self): super().__init__() self.stem = Stem(in_chs=3, mid_chs=32, out_chs=48) for index, stage in enumerate(HGNETV2_B4_STAGES): setattr( self, f"stages_{index}", DenseStage( in_chs=stage[0], mid_chs=stage[1], out_chs=stage[2], block_num=stage[3], layer_num=stage[7], downsample=stage[4], light_block=stage[5], kernel_size=stage[6], ), ) self.stage_names = tuple(f"stages_{index}" for index in range(len(HGNETV2_B4_STAGES))) class DeformableAdapter(nn.Module): """Identity-initialised bottleneck with one DCNv2 at its centre.""" def __init__(self, channels, bottleneck_channels): super().__init__() self.reduce = nn.Conv2d(channels, bottleneck_channels, 1, bias=False) self.bn1 = nn.BatchNorm2d(bottleneck_channels) self.dcn = DCNv2(bottleneck_channels, deform_groups=2) self.bn2 = nn.BatchNorm2d(bottleneck_channels) self.expand = nn.Conv2d(bottleneck_channels, channels, 1, bias=False) self.bn3 = nn.BatchNorm2d(channels) self.relu = nn.ReLU(inplace=True) def forward(self, x): residual = self.relu(self.bn1(self.reduce(x))) residual = self.relu(self.bn2(self.dcn(residual))) return x + self.bn3(self.expand(residual)) class HGNetv2B4Backbone(nn.Module): """HGNetv2-b4 plus the two deformable adapters.""" def __init__(self, deformable=True): super().__init__() self.deformable = deformable self.features = HGNetv2B4Features() self.out_channels = HGNetv2B4Features.out_channels self.dcn_adapters = nn.ModuleDict() if deformable: self.dcn_adapters["stages_1"] = DeformableAdapter(512, 128) self.dcn_adapters["stages_2"] = DeformableAdapter(1024, 256) def forward(self, images): if not self.deformable: return self.features(images) x = self.features.stem(images) for name in self.features.stage_names: x = getattr(self.features, name)(x) if name in self.dcn_adapters: x = self.dcn_adapters[name](x) return x class ConvAct(nn.Module): def __init__(self, in_channels, out_channels, kernel_size=3, activation=True): super().__init__() self.conv = nn.Conv2d(in_channels, out_channels, kernel_size, padding=kernel_size // 2) self.activate = nn.ReLU(inplace=True) if activation else nn.Identity() def forward(self, x): return self.activate(self.conv(x)) class NeckProjection(nn.Module): """Lateral conv then output conv (the only active FPN legs in the checkpoint).""" def __init__(self, in_channels, out_channels=256): super().__init__() self.lateral_convs = nn.ModuleList([ConvAct(in_channels, out_channels, 1, False)]) self.fpn_convs = nn.ModuleList([ConvAct(out_channels, out_channels, 3, False)]) def forward(self, x): return self.fpn_convs[0](self.lateral_convs[0](x)) class PolygonHead(nn.Module): """Dense/sparse class logits plus 14 low-rank polygon coefficients.""" def __init__(self, in_channels=256, num_coefficients=14, upsample=4): super().__init__() self.upsample = upsample self.cls_convs = nn.Sequential(ConvAct(in_channels, 32), ConvAct(32, 32)) self.out_conv_cls_dense = nn.Conv2d(32, 1, 3, padding=1) self.out_conv_reg_dense = nn.Conv2d(32, num_coefficients, 3, padding=1) self.out_conv_cls_sparse = nn.Conv2d(32, 1, 3, padding=1) self.out_conv_reg_sparse = nn.Conv2d(32, num_coefficients, 3, padding=1) def forward(self, feature): feature = F.interpolate(feature, scale_factor=self.upsample, mode="bilinear", align_corners=False) feature = self.cls_convs(feature) # ``out_conv_reg_dense`` is kept for checkpoint compatibility but never # read: decoding uses only the sparse branch. At 1600x1600 input this is # a 400x400x14 convolution that was pure waste. return { "dense_logits": self.out_conv_cls_dense(feature), "sparse_logits": self.out_conv_cls_sparse(feature), "sparse_coefficients": self.out_conv_reg_sparse(feature), } class DetectorNet(nn.Module): """Backbone, projection and head, in the order the frozen detector runs them.""" def __init__(self, num_coefficients=14, deformable=True, upsample=4): super().__init__() if upsample not in (1, 2, 4): raise ValueError("upsample must be 1, 2 or 4") self.output_stride = 16 // upsample self.backbone = HGNetv2B4Backbone(deformable=deformable) self.neck = NeckProjection(self.backbone.out_channels) self.bbox_head = PolygonHead(num_coefficients=num_coefficients, upsample=upsample) def forward(self, images): if images.ndim != 4 or any(size % 32 for size in images.shape[-2:]): raise ValueError("Expected BCHW images padded to multiples of 32") return self.bbox_head(self.neck(self.backbone(images))) def polygon_points(coefficients, basis, stride): """``[B,K,H,W]`` coefficients -> ``[B,H*W,P,2]`` pixel coords. """ batch, _, height, width = coefficients.shape basis = torch.as_tensor(basis, dtype=torch.float32, device=coefficients.device) points = coefficients.permute(0, 2, 3, 1).reshape(batch, height * width, -1) @ basis yy, xx = torch.meshgrid( torch.arange(height, device=coefficients.device, dtype=torch.float32), torch.arange(width, device=coefficients.device, dtype=torch.float32), indexing="ij", ) locations = torch.stack((xx.reshape(-1), yy.reshape(-1)), dim=1) locations = locations.repeat(1, basis.shape[1] // 2) return (points + locations) * stride @torch.no_grad() def _drop_degenerate(polygons, content_shape, config): """Mask out boxes too small to hold readable text. The optional area cap is disabled by default -- see the comment below for why "covers half the image" is not a reliable signal. """ flat = polygons.reshape(len(polygons), -1, 2) wh = (flat.amax(dim=1) - flat.amin(dim=1)).clamp(min=0) min_side = float(config.get("min_box_side", 4.0)) keep = (wh[:, 0] >= min_side) & (wh[:, 1] >= min_side) # Area cap is OFF by default: a single-line crop legitimately fills most of # its own image, so "covers half the page" is not evidence of a bad box. # The few page-sized regions it would catch are not worth deleting whole # lines from cropped inputs. fraction = float(config.get("max_box_area_fraction", 1.0)) if content_shape is not None and fraction < 1.0: height, width = content_shape keep = keep & ((wh[:, 0] * wh[:, 1]) <= fraction * float(height * width)) return keep def _suppress_contained(polygons, scores, contain=0.95, ratio=0.35): """Drop boxes nested inside a larger box. Measured on 3544 pages: 54% of images contain nested boxes and 40% of them repeat text already returned by the enclosing line. Containment (not IoU) is the right test -- a small box inside a large one has a low IoU, so an IoU filter never fires. The ratio guard keeps legitimate nesting: a heading inside a paragraph block is usually more than a third of its area, so it survives. """ n = len(polygons) if n < 2: return polygons, scores flat = polygons.reshape(n, -1, 2) lo, hi = flat.amin(dim=1), flat.amax(dim=1) areas = (hi - lo).clamp(min=0).prod(dim=1) order = torch.argsort(areas, descending=True) lo, hi, areas = lo[order], hi[order], areas[order] alive = torch.ones(n, dtype=torch.bool, device=areas.device) for pos in range(n - 1): if not bool(alive[pos]): continue candidate = alive[pos + 1:] if not bool(candidate.any()): continue c_lo, c_hi, c_area = lo[pos + 1:], hi[pos + 1:], areas[pos + 1:] ix0 = torch.maximum(lo[pos][0], c_lo[:, 0]) iy0 = torch.maximum(lo[pos][1], c_lo[:, 1]) ix1 = torch.minimum(hi[pos][0], c_hi[:, 0]) iy1 = torch.minimum(hi[pos][1], c_hi[:, 1]) inter = (ix1 - ix0).clamp(min=0) * (iy1 - iy0).clamp(min=0) nested = (inter / c_area.clamp(min=1e-6) >= contain) & ( c_area / areas[pos].clamp(min=1e-6) <= ratio ) alive[pos + 1:] = candidate & ~nested keep = torch.zeros(n, dtype=torch.bool, device=areas.device) keep[order] = alive return polygons[keep], scores[keep] def decode_detection(predictions, basis, config, content_shape=None, scale=None): """Scores plus polygons to accepted boxes, mirroring the frozen decoder. ``config`` needs ``stride``, ``score_threshold``, ``nms_threshold`` and ``max_candidates``. ``content_shape`` (h, w pre-padding) masks padding cells; ``scale`` divides polygons back to original image coordinates. """ stride = config["stride"] points = polygon_points( predictions["sparse_coefficients"].float(), basis, stride ) confidence = ( predictions["dense_logits"].sigmoid() * predictions["sparse_logits"].sigmoid() )[:, 0] if content_shape is not None: height, width = content_shape confidence = confidence.clone() confidence[:, (height + stride - 1) // stride :] = 0 confidence[:, :, (width + stride - 1) // stride :] = 0 results = [] for polygons, scores in zip(points, confidence.flatten(1)): selected = (scores > config["score_threshold"]).nonzero(as_tuple=True)[0] if selected.numel() > config["max_candidates"]: selected = selected[ scores[selected].topk(config["max_candidates"]).indices ] polygons, scores = polygons[selected], scores[selected] if polygons.numel(): flat = polygons.reshape(len(polygons), -1, 2) boxes = torch.cat((flat.amin(dim=1), flat.amax(dim=1)), dim=1) keep = nms(boxes, scores, config["nms_threshold"]) polygons, scores = polygons[keep], scores[keep] # Added post-processing, all geometry-only so it runs before recognition. keep = _drop_degenerate(polygons, content_shape, config) polygons, scores = polygons[keep], scores[keep] if polygons.numel() and config.get("deduplicate_nested", False): polygons, scores = _suppress_contained( polygons, scores, float(config.get("contain_threshold", 0.95)), float(config.get("contain_area_ratio", 0.35)), ) if content_shape is not None and polygons.numel(): # Clamp to the content area, not the padded canvas: a polygon # that runs into the padding would otherwise be cropped with # replicated border pixels. content_h, content_w = content_shape view = polygons.reshape(len(polygons), -1, 2) view[..., 0].clamp_(0, content_w - 1) view[..., 1].clamp_(0, content_h - 1) if scale is not None: polygons = polygons / torch.as_tensor( scale, dtype=polygons.dtype, device=polygons.device ).repeat(polygons.shape[-1] // 2) results.append( {"polygons": polygons.cpu().numpy(), "scores": scores.cpu().numpy()} ) return results MEAN = np.array([123.675, 116.28, 103.53], dtype=np.float32) STD = np.array([58.395, 57.12, 57.375], dtype=np.float32) def detection_tensor(rgb, size=1600): """Letterbox one RGB uint8 image to ``size`` and normalise. Returns ``(tensor [1,3,size,size] float32, scale [2], content (h, w))``. """ height, width = rgb.shape[:2] scale = size / max(height, width) new_w = max(1, round(width * scale)) new_h = max(1, round(height * scale)) resized = np.asarray(Image.fromarray(rgb).resize((new_w, new_h), Image.BILINEAR)) canvas = np.zeros((size, size, 3), np.float32) canvas[:new_h, :new_w] = (resized.astype(np.float32) - MEAN) / STD tensor = np.ascontiguousarray(canvas.transpose(2, 0, 1)[None]) return tensor, np.array([new_w / width, new_h / height], np.float32), (new_h, new_w) def _require_cv2(): """Import OpenCV on demand. ``transformers`` statically scans a ``trust_remote_code`` module for imports and refuses to load it when any of them is missing, so a top-level ``import cv2`` would make the detector unloadable on a machine that only wants ``detect()``. The import sits inside a ``try`` block because the scanner does not descend into those. """ try: import cv2 except ImportError as exc: # pragma: no cover - depends on the environment raise ImportError( "OpenCV is required for polygon rectification (det.ocr / the crop " "step). Install it with `pip install opencv-python-headless` -- the " "transformers hint `pip install cv2` refers to a package that does " "not exist. Detection alone (det.detect) works without it." ) from exc return cv2 def polygon_to_quad(polygon, image_shape): """Return TL/TR/BR/BL float32 corners compatible with OpenOCR's cropper. Uses OpenOCR's minAreaRect corner convention (sort by x, then split into left/right pairs), without the int16 coordinate limit. """ points = np.asarray(polygon, dtype=np.float32).reshape(-1, 2).copy() height, width = image_shape[:2] if len(points) < 3 or not np.isfinite(points).all() or min(height, width) < 2: return None cv2 = _require_cv2() points[:, 0] = np.clip(points[:, 0], 0, width - 1) points[:, 1] = np.clip(points[:, 1], 0, height - 1) rect = cv2.minAreaRect(points) if min(rect[1]) < 2: return None corners = cv2.boxPoints(rect) corners = corners[np.argsort(corners[:, 0], kind="stable")] left = corners[:2][np.argsort(corners[:2, 1], kind="stable")] right = corners[2:][np.argsort(corners[2:, 1], kind="stable")] quad = np.asarray([left[0], right[0], right[1], left[1]], dtype=np.float32) quad[:, 0] = np.clip(quad[:, 0], 0, width - 1) quad[:, 1] = np.clip(quad[:, 1], 0, height - 1) crop_width = max( np.linalg.norm(quad[0] - quad[1]), np.linalg.norm(quad[2] - quad[3]) ) crop_height = max( np.linalg.norm(quad[0] - quad[3]), np.linalg.norm(quad[1] - quad[2]) ) if min(crop_width, crop_height) < 2 or abs(cv2.contourArea(quad)) < 1: return None return quad def get_rotate_crop_image(img, points): """Four-point perspective crop, identical to the frozen OpenOCR cropper. Tall crops (height / width >= 1.5) are rotated 90 degrees so vertical text is fed horizontally; dropping that step costs accuracy on vertical samples. """ points = np.asarray(points, dtype=np.float32).reshape(4, 2) cv2 = _require_cv2() img_crop_width = int( max(np.linalg.norm(points[0] - points[1]), np.linalg.norm(points[2] - points[3])) ) img_crop_height = int( max(np.linalg.norm(points[0] - points[3]), np.linalg.norm(points[1] - points[2])) ) pts_std = np.float32( [ [0, 0], [img_crop_width, 0], [img_crop_width, img_crop_height], [0, img_crop_height], ] ) matrix = cv2.getPerspectiveTransform(points, pts_std) cropped = cv2.warpPerspective( img, matrix, (img_crop_width, img_crop_height), borderMode=cv2.BORDER_REPLICATE, flags=cv2.INTER_CUBIC, ) height, width = cropped.shape[0:2] if width and height * 1.0 / width >= 1.5: cropped = np.rot90(cropped) return cropped class OCRPipeline: """Detector plus one recogniser, in the frozen pipeline's order. Detection runs per image, polygons become upright quads, crops are sorted and bucketed by recognition shape, and regions come back in reading order (top-to-bottom, then left-to-right of each quad's top-left corner). """ def __init__(self, det_model, rec_model, batch_size=32, drop_score=0.0): self.det_model = det_model self.rec_model = rec_model self.batch_size = batch_size self.drop_score = drop_score def __call__(self, image, color="rgb"): """``image`` is a path, bytes, PIL image or HxWx3 array. Returns ``{"text": [str], "regions": [{text, score, quad, ...}]}``. """ return self.run(self._read(image, color)) def run(self, rgb): regions = self.detect(rgb) if not regions: return {"text": [], "regions": []} # ``recognition_batches`` expects BGR crops and converts them to RGB # itself, so the crop stays in BGR rather than being flipped back here. bgr = np.ascontiguousarray(rgb[:, :, ::-1]) crops = [get_rotate_crop_image(bgr, region["quad"]) for region in regions] recognised = self.recognize(crops) keep = [] for region, (text, score) in zip(regions, recognised): if score is not None and not np.isnan(score) and score < self.drop_score: continue region = dict(region) region["text"] = text region["rec_score"] = None if score is None else float(score) keep.append(region) return {"text": [region["text"] for region in keep], "regions": keep} def detect(self, rgb): raw = self.det_model.detect([rgb], batch_size=1)[0] regions = [] for polygon, score in zip(raw["polygons"], raw["scores"]): quad = polygon_to_quad(polygon, rgb.shape) if quad is None: continue regions.append( {"polygon": np.asarray(polygon), "quad": quad, "det_score": float(score)} ) return sorted(regions, key=lambda row: (row["quad"][0, 1], row["quad"][0, 0])) def recognize(self, crops): return self.rec_model.recognize(crops, batch_size=self.batch_size) @staticmethod def _read(image, color): from PIL import Image, ImageOps def _to_rgb(im): """Composite alpha over white, apply EXIF orientation, return RGB.""" im = ImageOps.exif_transpose(im) if im.mode in ("RGBA", "LA") or "A" in im.mode: bg = Image.new("RGBA", im.size, (255, 255, 255, 255)) im = Image.alpha_composite(bg, im.convert("RGBA")).convert("RGB") elif im.mode == "P": im = im.convert("RGBA") bg = Image.new("RGBA", im.size, (255, 255, 255, 255)) im = Image.alpha_composite(bg, im).convert("RGB") else: im = im.convert("RGB") return im if isinstance(image, np.ndarray): if image.ndim == 2: image = np.stack([image] * 3, axis=-1) if color == "bgr": return np.ascontiguousarray(image[:, :, ::-1]) return np.ascontiguousarray(image) if isinstance(image, (bytes, bytearray)): import io with Image.open(io.BytesIO(image)) as im: return np.asarray(_to_rgb(im)) if isinstance(image, (str, bytes)) or hasattr(image, "__fspath__"): with Image.open(image) as im: return np.asarray(_to_rgb(im)) return np.asarray(_to_rgb(image)) class OpenOCRv2DetConfig(PretrainedConfig): """Configuration for the OpenOCRv2 detector (``OpenOCRv2_Det``). The defaults are the frozen post-processing constants from ``models/manifest.json`` (``models.det``); overriding them changes which boxes survive, not just the scores. """ model_type = "openocrv2_det" def __init__( self, image_size=1600, stride=4, score_threshold=0.2, nms_threshold=0.5, max_candidates=2000, num_coefficients=14, upsample=4, deformable=True, architecture="hgnetv2_b4", **kwargs, ): super().__init__(**kwargs) self.image_size = image_size self.stride = stride self.score_threshold = score_threshold self.nms_threshold = nms_threshold self.max_candidates = max_candidates self.num_coefficients = num_coefficients self.upsample = upsample self.deformable = deformable self.architecture = architecture @property def decode_config(self): return { "stride": self.stride, "score_threshold": self.score_threshold, "nms_threshold": self.nms_threshold, "max_candidates": self.max_candidates, # Nested-box suppression is OFF by default. It removes text that a # downstream metric may expect, and ANLS never rewards removing it # (measured: -0.43 points on a 54-image ablation). Turn it on for # pseudo-labelling, where the goal is clean boxes for annotators. "deduplicate_nested": getattr(self, "deduplicate_nested", False), "contain_threshold": getattr(self, "contain_threshold", 0.95), "contain_area_ratio": getattr(self, "contain_area_ratio", 0.35), "min_box_side": getattr(self, "min_box_side", 4.0), } @dataclass class OpenOCRv2DetOutput(ModelOutput): """Raw detection head outputs plus absolute pixel polygons.""" dense_logits: torch.FloatTensor | None = None sparse_logits: torch.FloatTensor | None = None sparse_coefficients: torch.FloatTensor | None = None class OpenOCRv2DetModel(PreTrainedModel): """OpenOCRv2 detector: dense/sparse logits plus 14 polygon coefficients.""" config_class = OpenOCRv2DetConfig main_input_name = "pixel_values" def __init__(self, config): super().__init__(config) self.model = DetectorNet( num_coefficients=config.num_coefficients, deformable=config.deformable, upsample=config.upsample, ) # Polygon basis (14 x 28). A buffer, so it travels with the checkpoint. self.register_buffer( "basis", torch.zeros(config.num_coefficients, config.num_coefficients * 2) ) self.post_init() def forward(self, pixel_values, **kwargs): predictions = self.model(pixel_values) return OpenOCRv2DetOutput( dense_logits=predictions["dense_logits"], sparse_logits=predictions["sparse_logits"], sparse_coefficients=predictions["sparse_coefficients"], ) def detect(self, images, batch_size=1): """Run detection on a list of RGB uint8 arrays. Returns one dict per image with ``polygons``, ``quads``, ``scores`` and the polygons expressed in original-image pixels. """ self.eval() results = [] device = next(self.parameters()).device for start in range(0, len(images), batch_size): chunk = images[start : start + batch_size] tensors, scales, contents = [], [], [] for rgb in chunk: tensor, scale, content = detection_tensor( rgb, self.config.image_size ) tensors.append(tensor[0]) scales.append(scale) contents.append(content) pixel_values = torch.from_numpy(np.stack(tensors)).to(device) with torch.no_grad(): outputs = self(pixel_values=pixel_values) for index, (scale, content) in enumerate(zip(scales, contents)): single = { "dense_logits": outputs.dense_logits[index : index + 1], "sparse_logits": outputs.sparse_logits[index : index + 1], "sparse_coefficients": outputs.sparse_coefficients[index : index + 1], } decoded = decode_detection( single, self.basis, self.config.decode_config, content_shape=content, scale=scale, )[0] results.append(decoded) return results def ocr(self, image, rec_model, batch_size=32, drop_score=0.0): """End-to-end OCR on one page; ``rec_model`` is one of the recognizers. Detect, rectify every polygon into an upright crop, recognise, and return the regions in reading order. This is the same sequence as ``OCRPipeline`` -- it just saves the caller from tying the two halves together (and from re-implementing the cropper, which is where accuracy is most easily lost). ``image`` is a path, bytes, PIL image or HxWx3 uint8 array. Returns ``{"text": [str], "regions": [{text, rec_score, det_score, quad}]}``. """ return OCRPipeline( self, rec_model, batch_size=batch_size, drop_score=drop_score )(image)